How to use imshow() in Mobile App on Kotlin?

Question

I decided to create a computer vision model that detect an object. It takes a real-time image from camera and show rectangles of detected objects. But I dunno how to visualize OpenCV imshow() function.

My model use Yolov8 and I tried to convert it to tflite, but Tensorflow has been updated, so for now it's impossible. So one way to deploy model on mobile app is to send results on the screen. But I don't know Kotlin and for me it's too hard

Answer 1

Steps to Run YOLO Model on Mobile using Kotlin

1. nConvert YOLOv8 to ONNX format.
2. Use ONNX Runtime in your Android application to run the model.
3. Capture image using Camera and process it through the model.
4. Draw rectangles on detected objects on the captured image.

dependencies {
    implementation 'com.microsoft.onnxruntime:onnxruntime-android:1.8.1'
}

Load the ONNX model in your Kotlin code:

import ai.onnxruntime.*

class YoloV8Model(context: Context) {

    private val ortEnvironment: OrtEnvironment = OrtEnvironment.getEnvironment()
    private val sessionOptions = OrtSession.SessionOptions()
    private val ortSession: OrtSession

    init {
        val modelBytes = context.assets.open("yolov8.onnx").readBytes()
        ortSession = ortEnvironment.createSession(modelBytes, sessionOptions)
    }

    fun detectObjects(bitmap: Bitmap): List<Detection> {
        val resizedBitmap = Bitmap.createScaledBitmap(bitmap, 640, 640, true)
        val floatBuffer = convertBitmapToFloatBuffer(resizedBitmap)

        val inputName = ortSession.inputNames.iterator().next()
        val inputTensor = OnnxTensor.createTensor(ortEnvironment, floatBuffer, longArrayOf(1, 3, 640, 640))

        val results = ortSession.run(mapOf(inputName to inputTensor))
        val outputTensor = results[0].value as Array<FloatArray>

        return parseDetections(outputTensor)
    }

    private fun convertBitmapToFloatBuffer(bitmap: Bitmap): FloatBuffer {
        val floatBuffer = FloatBuffer.allocate(3 * 640 * 640)
        val intValues = IntArray(640 * 640)
        bitmap.getPixels(intValues, 0, 640, 0, 0, 640, 640)

        for (pixelValue in intValues) {
            floatBuffer.put(((pixelValue shr 16) and 0xFF) / 255.0f)
            floatBuffer.put(((pixelValue shr 8) and 0xFF) / 255.0f)
            floatBuffer.put((pixelValue and 0xFF) / 255.0f)
        }

        floatBuffer.rewind()
        return floatBuffer
    }

    private fun parseDetections(outputTensor: Array<FloatArray>): List<Detection> {
        val detections = mutableListOf<Detection>()
        for (detection in outputTensor) {
            val confidence = detection[4]
            if (confidence > 0.5) {
                val x1 = detection[0]
                val y1 = detection[1]
                val x2 = detection[2]
                val y2 = detection[3]
                val classId = detection[5].toInt()
                detections.add(Detection(x1, y1, x2, y2, confidence, classId))
            }
        }
        return detections
    }
}

data class Detection(val x1: Float, val y1: Float, val x2: Float, val y2: Float, val confidence: Float, val classId: Int)

Use the YoloV8Model class to run the model and draw rectangles on the detected objects:

import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Paint

fun drawDetections(bitmap: Bitmap, detections: List<Detection>): Bitmap {
    val mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true)
    val canvas = Canvas(mutableBitmap)
    val paint = Paint().apply {
        color = Color.RED
        strokeWidth = 3f
        style = Paint.Style.STROKE
    }

    for (detection in detections) {
        canvas.drawRect(detection.x1, detection.y1, detection.x2, detection.y2, paint)
    }

    return mutableBitmap
}

Capture the image using Camera and process it through the model:

import android.graphics.BitmapFactory
import android.os.Bundle
import androidx.appcompat.app.AppCompatActivity
import kotlinx.android.synthetic.main.activity_main.*

class MainActivity : AppCompatActivity() {

    private lateinit var yoloV8Model: YoloV8Model

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_main)

        yoloV8Model = YoloV8Model(this)

        // Capture image (assuming you have an image in assets for demo purposes)
        val inputStream = assets.open("test_image.jpg")
        val bitmap = BitmapFactory.decodeStream(inputStream)

        val detections = yoloV8Model.detectObjects(bitmap)
        val resultBitmap = drawDetections(bitmap, detections)

        imageView.setImageBitmap(resultBitmap)
    }
}