Search code examples
javadrawingobject-detectionandroid-cameraxgoogle-mlkit

Why isn't my graphic overlay appearing on input image?


My application using CameraX and Google ML Kit written in Java. The purpose of the application is to detect objects with a real time camera preview. I implemented ML Kit using this guide aptly titled "Detect and track objects with ML Kit on Android" (base model option) to detect objects in successive frames within the application. Despite creating a graphic overlay to draw a bounding box, I'm confused as to why it is not appearing on my user interface upon launching the application onto my device. Here is the code;

MainActivity.java

public class MainActivity extends AppCompatActivity  {

    AlertDialog alertDialog;

    private ListenableFuture<ProcessCameraProvider> cameraProviderFuture;

    private class YourAnalyzer implements ImageAnalysis.Analyzer {

        @Override
        @ExperimentalGetImage
        @SuppressLint("UnsafeExperimentalUsageError")
        public void analyze(ImageProxy imageProxy) {
            Image mediaImage = imageProxy.getImage();
            if (mediaImage != null) {
                //Log.d("TAG", "mediaImage is throwing null");
                InputImage image =
                        InputImage.fromMediaImage(mediaImage, imageProxy.getImageInfo().getRotationDegrees());
                //Pass image to an ML Kit Vision API
                //...

                ObjectDetectorOptions options =
                        new ObjectDetectorOptions.Builder()
                                .setDetectorMode(ObjectDetectorOptions.STREAM_MODE)
                                .enableClassification()  // Optional
                                .build();

                ObjectDetector objectDetector = ObjectDetection.getClient(options);

                objectDetector.process(image)
                        .addOnSuccessListener(detectedObjects -> {
                            getObjectResults(detectedObjects);
                            Log.d("TAG", "onSuccess" + detectedObjects.size());
                            for (DetectedObject detectedObject : detectedObjects) {
                                Rect boundingBox = detectedObject.getBoundingBox();

                                Integer trackingId = detectedObject.getTrackingId();
                                for (DetectedObject.Label label : detectedObject.getLabels()) {
                                    String text = label.getText();
                                    int index = label.getIndex();
                                    float confidence = label.getConfidence();
                                }
                            }
                        })
                        .addOnFailureListener(e -> Log.e("TAG", e.getLocalizedMessage()))
                        .addOnCompleteListener(result -> imageProxy.close());

            }

        }
    }

    private void getObjectResults(List<DetectedObject> detectedObjects) {
        int count=0;
        for (DetectedObject object:detectedObjects)
        {
            Rect rect = object.getBoundingBox();
            String text = "Undefined";
            DrawGraphic drawGraphic = new DrawGraphic(this, rect, text);
            count = count+1;
        }
        alertDialog.dismiss();
    }

    @Override
    protected void onCreate(@Nullable Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        cameraProviderFuture = ProcessCameraProvider.getInstance(this);

        PreviewView previewView = findViewById(R.id.previewView);

        alertDialog = new SpotsDialog.Builder()
                .setContext(this)
                .setMessage("Currently processing...")
                .setCancelable(false)
                .build();

        cameraProviderFuture.addListener(() -> {
            try {
                ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
                bindPreview(cameraProvider);
            } catch (ExecutionException | InterruptedException e) {
                // No errors need to be handled for this Future.
                // This should never be reached.
            }
        }, ContextCompat.getMainExecutor(this));
    }

    void bindPreview(@NonNull ProcessCameraProvider cameraProvider) {

        PreviewView previewView = findViewById(R.id.previewView);
        Preview preview = new Preview.Builder()
                .build();

        CameraSelector cameraSelector = new CameraSelector.Builder()
                .requireLensFacing(CameraSelector.LENS_FACING_BACK)
                .build();

        preview.setSurfaceProvider(previewView.getSurfaceProvider());

        ImageAnalysis imageAnalysis =
                new ImageAnalysis.Builder()
                        .setTargetResolution(new Size(1280,720))
                        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
                        .build();
        imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this), new YourAnalyzer());

        Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview, imageAnalysis);
    }
}

DrawGraphic.java

public class DrawGraphic extends View {

    Paint borderPaint, textPaint;
    Rect rect;
    String text;

    public DrawGraphic(Context context, Rect rect, String text) {
        super(context);
        this.rect = rect;
        this.text = text;

        borderPaint = new Paint();
        borderPaint.setColor(Color.WHITE);
        borderPaint.setStrokeWidth(10f);
        borderPaint.setStyle(Paint.Style.STROKE);

        textPaint = new Paint();
        textPaint.setColor(Color.WHITE);
        textPaint.setStrokeWidth(50f);
        textPaint.setTextSize(32f);
        textPaint.setStyle(Paint.Style.FILL);
    }

    @Override
    protected void onDraw(Canvas canvas) {
        super.onDraw(canvas);
        canvas.drawText(text, rect.centerX(), rect.centerY(), textPaint);
        canvas.drawRect(rect.left, rect.top, rect.right, rect.bottom, borderPaint);
    }
}

The end objective of my code is that I want objects to be detected in real time like so;

enter image description here

Any information needed to supplement this question will be provided upon request.


Solution

  • I was able to resolve this issue by enabling viewBinding with the help of this Android documentation because it allows one to more easily write code that interacts with views. I made changes in my Gradle file like so;

    android {
        buildFeatures {
            viewBinding true
        }
    }
    

    Then, I added these changes to my activity_main.xml;

    <RelativeLayout
        xmlns:android="http://schemas.android.com/apk/res/android"
        xmlns:app="http://schemas.android.com/apk/res-auto"
        xmlns:tools="http://schemas.android.com/tools"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        tools:context=".MainActivity">
        
        <FrameLayout
            android:id="@+id/parentlayout"
            android:layout_width="match_parent"
            android:layout_height="match_parent">
    
            <androidx.camera.view.PreviewView
                android:id="@+id/previewView"
                android:layout_height="match_parent"
                android:layout_width="match_parent"/>
        </FrameLayout>
    
    </RelativeLayout>
    

    I then turned my attention to MainActivity.java, declaring ActivityMainBinding binding; as an attribute within the class, and then reformatting the following method using this Github user's result processing method as a reference like so;

        private void getObjectResults(List<DetectedObject> detectedObjects) {
            for (DetectedObject object : detectedObjects) {
                if (binding.parentlayout.getChildCount() > 1) {
                    binding.parentlayout.removeViewAt(1);
                }
                Rect rect = object.getBoundingBox();
                String text = "Undefined";
                if (object.getLabels().size() != 0) {
                    text = object.getLabels().get(0).getText();
                }
    
                DrawGraphic drawGraphic = new DrawGraphic(this, rect, text);
                binding.parentlayout.addView(drawGraphic);
            }
            /*alertDialog.dismiss();*/
        }