Search code examples
javaandroidkotlinandroid-cameraxgoogle-mlkit

Text Recognition Overlay Misalignment Issue in Camera Preview

I'm working on an Android application where I'm implementing live text recognition using CameraX and ML Kit. The recognized text is displayed with bounding boxes on the camera preview, but I'm facing an issue where these bounding boxes are not aligning correctly with the text in the live feed.

Problem Description When running the application:

  • The camera preview displays the live feed correctly.
  • The ML Kit Text Recognition processes the image and identifies text blocks.
  • Bounding boxes are drawn around detected text elements.
  • However, these bounding boxes do not align with the actual text in the camera preview. They appear displaced or incorrectly scaled.

Main Activity Snippets

package com.aviskaarlab.booksnap.ui.views.home

import android.Manifest
import android.content.pm.PackageManager
import android.os.Build
import android.os.Bundle
import android.util.Log
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.AspectRatio
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.Preview
import androidx.camera.core.resolutionselector.AspectRatioStrategy
import androidx.camera.core.resolutionselector.ResolutionSelector
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors

class MainActivity : AppCompatActivity() {
    private lateinit var viewBinding: ActivityMainBinding
    private lateinit var cameraExecutor: ExecutorService
    private lateinit var textOverlay: TextOverlay
    private lateinit var viewFinder: PreviewView

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        viewBinding = ActivityMainBinding.inflate(layoutInflater)
        setContentView(viewBinding.root)

        viewFinder = viewBinding.previewView
        textOverlay = viewBinding.textOverlay

        // Initialize camera and start preview
        startCamera()

        cameraExecutor = Executors.newSingleThreadExecutor()
    }

    private fun startCamera() {
        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)

        cameraProviderFuture.addListener({
            val resolutionSelector = ResolutionSelector.Builder()
                .setAspectRatioStrategy(AspectRatioStrategy.RATIO_4_3_FALLBACK_AUTO_STRATEGY)
                .build()

            val rotation = viewFinder.display.rotation
            val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()

            // Preview
            val preview = Preview.Builder()
                .setResolutionSelector(resolutionSelector)
                .setTargetRotation(rotation)
                .build()
                .also {
                    it.setSurfaceProvider(viewBinding.previewView.surfaceProvider)
                }

            // Image Analysis
            val imageAnalysis = ImageAnalysis.Builder()
                .setResolutionSelector(resolutionSelector)
                .setTargetRotation(rotation)
                .build()
                .also {
                    it.setAnalyzer(
                        cameraExecutor,
                        BookSnapWordAnalyzer(
                            textOverlay,
                            viewBinding.previewView,
                        )
                    )
                }

            // Select back camera as default
            val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA

            try {
                cameraProvider.unbindAll()
                cameraProvider.bindToLifecycle(
                    this, cameraSelector, preview, imageAnalysis
                )

                preview.setSurfaceProvider(viewFinder.surfaceProvider)
            } catch (exc: Exception) {
                Log.e(TAG, "Use case binding failed", exc)
            }

        }, ContextCompat.getMainExecutor(this))
    }

    override fun onDestroy() {
        super.onDestroy()
        cameraExecutor.shutdown()
    }

    companion object {
        private const val TAG = "CameraXApp"
    }
}

BookSnapWordAnalyzer Code Snippet

package com.aviskaarlab.booksnap.ui.views.home

import android.graphics.Matrix
import android.graphics.Rect
import android.graphics.RectF
import android.util.Log
import androidx.camera.core.ExperimentalGetImage
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import androidx.camera.view.PreviewView
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.text.Text
import com.google.mlkit.vision.text.TextRecognition
import com.google.mlkit.vision.text.latin.TextRecognizerOptions

internal class BookSnapWordAnalyzer(
    private val overlay: TextOverlay,
    private val previewView: PreviewView,
) : ImageAnalysis.Analyzer {

    companion object {
        private const val TAG = "BookSnapWordAnalyzer"
        private const val WORD_LENGTH = 4
    }

    private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
    private lateinit var visionText: Text
    private lateinit var matrix: Matrix
    private var rotationDegrees: Int = 0

    @OptIn(ExperimentalGetImage::class)
    override fun analyze(imageProxy: ImageProxy) {
        val mediaImage = imageProxy.image ?: return
        rotationDegrees = imageProxy.imageInfo.rotationDegrees
        matrix = getCorrectionMatrix(imageProxy, previewView)

        val image =
            InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)

        recognizer.process(image)
            .addOnSuccessListener { visionText ->
                this.visionText = visionText
                val boxes = mutableListOf<CustomRect>()
                for (block in visionText.textBlocks) {
                    for (line in block.lines) {
                        for (element in line.elements) {
                            val elementText = element.text
                            val boundingBox = element.boundingBox
                            if (elementText.length >= WORD_LENGTH && boundingBox != null) {
                                boxes.add(
                                    CustomRect(
                                        adjustBoundingBox(boundingBox, imageProxy, previewView),
                                        elementText
                                    )
                                )
                            }
                        }
                    }
                }
                overlay.updateBoundingBoxes(boxes)
            }
            .addOnFailureListener { e ->
                Log.e(TAG, "Text recognition failed", e)
            }.addOnCompleteListener {
                imageProxy.close()
            }
    }

    private fun adjustBoundingBox(
        rect: Rect,
        imageProxy: ImageProxy,
        previewView: PreviewView
    ): RectF {
        val cropRect = imageProxy.cropRect
        val imageWidth = cropRect.width()
        val imageHeight = cropRect.height()

        val previewWidth = previewView.width
        val previewHeight = previewView.height

        val scaleX = previewWidth.toFloat() / imageWidth
        val scaleY = previewHeight.toFloat() / imageHeight

        val verticalOffset = (previewHeight - imageHeight * scaleY) / 2
        val horizontalOffset = (previewWidth - imageWidth * scaleX) / 2

        val left = rect.left * scaleX + horizontalOffset
        val top = rect.top * scaleY + verticalOffset
        val right = rect.right * scaleX + horizontalOffset
        val bottom = rect.bottom * scaleY + verticalOffset

        return RectF(left, top, right, bottom)
    }

    private fun getCorrectionMatrix(
        imageProxy: ImageProxy,
        previewView: PreviewView,
    ): Matrix {
        val cropRect = imageProxy.cropRect
        val matrix = Matrix()

        val source = floatArrayOf(
            cropRect.left.toFloat(), cropRect.top.toFloat(),
            cropRect.right.toFloat(), cropRect.top.toFloat(),
            cropRect.right.toFloat(), cropRect.bottom.toFloat(),
            cropRect.left.toFloat(), cropRect.bottom.toFloat()
        )

        val destination = floatArrayOf(
            0f, 0f,
            previewView.width.toFloat(), 0f,
            previewView.width.toFloat(), previewView.height.toFloat(),
            0f, previewView.height.toFloat()
        )

        matrix.setPolyToPoly(source, 0, destination, 0, 4)
        return matrix
    }
}

TextOverlay Code Snippet

package com.aviskaarlab.booksnap.ui.views.home

import android.content.Context
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Paint
import android.graphics.RectF
import android.util.AttributeSet
import android.view.MotionEvent
import android.view.View

class TextOverlay @JvmOverloads constructor(
    context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0
) : View(context, attrs, defStyleAttr) {

    private val paint = Paint().apply {
        color = Color.RED
        style = Paint.Style.STROKE
        strokeWidth = 2.0f
    }

    private val boundingBoxes = mutableListOf<CustomRect>()
    private var clickListener: ((String) -> Unit)? = null

    fun setOnRectangleClickListener(listener: (String) -> Unit) {
        clickListener = listener
    }

    fun updateBoundingBoxes(newBoundingBoxes: List<CustomRect>) {
        boundingBoxes.clear()
        boundingBoxes.addAll(newBoundingBoxes)
        invalidate() // Redraw the view
    }

    override fun onDraw(canvas: Canvas) {
        super.onDraw(canvas)
        for (box in boundingBoxes) {
            canvas.drawRect(box.rect, paint)
        }
    }

    override fun onTouchEvent(event: MotionEvent): Boolean {
        if (event.action == MotionEvent.ACTION_UP) {
            val x = event.x
            val y = event.y
            for (box in boundingBoxes) {
                if (box.rect.contains(x, y)) {
                    clickListener?.invoke("Clicked on rectangle ${box.text}")
                    return true
                }
            }
        }
        return true // Event handled
    }
}

Issue The rectangles drawn around recognized text do not align with the text in the camera preview. How can I adjust the bounding box coordinates so that they correctly overlay on the text in the live camera feed?(Please see image attached)

This is the issue, boxes are not aligned with the text

Solution

  • If you are using MLKit and PreviewView, use MLKitAnalyzer to detect image while setting the target coordinate system to COORDINATE_SYSTEM_VIEW_REFERENCED. The output coordinates will be in the PreviewView coordinate system.