Surf image matcher this with distorted results

Question

I have the following class that uses the android camera to capture an information to make a match of another image using opencv 3.1 with surf.

public class MyCameraPreview implements SurfaceHolder.Callback, Camera.PreviewCallback {

    private Camera mCamera = null;
    private ImageView ivCameraPreview = null;
    private int[] pixels;
    private Mat matFrameCamera;
    private Mat matFrameOutputCamera;
    private Mat mRgba2Gray;
    private int imageFormat;
    private int previewSizeWidth;
    private int previewSizeHeight;
    private boolean bProcessing = false;

    private int totalFrames = 0;
    private int totalProcFrames = 0;

    private Mat matTarget;
    private Bitmap bmpFixedImage;
    private Bitmap bitmap;

    private Handler mHandler = new Handler(Looper.getMainLooper());
    private Context context;

    private boolean findFeaturesMatch;
    private boolean surfCompletedWithMatcher;
    private boolean stop;

    MyCameraPreview(int previewLayoutWidth, int previewLayoutHeight,
                  ImageView ivCameraPreview, Context context) {

        previewSizeWidth = previewLayoutWidth;
        previewSizeHeight = previewLayoutHeight;
        this.ivCameraPreview = ivCameraPreview;
        this.context = context;

        try {
            matTarget = Utils.loadResource(context, R.drawable.outback, Imgcodecs.CV_LOAD_IMAGE_GRAYSCALE);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    @Override
    public void onPreviewFrame(byte[] arg0, Camera arg1) {    }

    void onPause() {
        mCamera.stopPreview();
    }

    @Override
    public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) {

        Camera.Parameters parameters = mCamera.getParameters();

        matFrameCamera = new Mat(previewSizeHeight + (previewSizeHeight / 2), previewSizeWidth, CvType.CV_8UC1);
        matFrameOutputCamera = new Mat(previewSizeHeight, previewSizeWidth, CvType.CV_8UC4);

        Camera.Size size = parameters.getPreviewSize();
        previewSizeHeight = size.height;
        previewSizeWidth = size.width;

        // Set the camera preview size
        parameters.setPreviewSize(previewSizeWidth, previewSizeHeight);

        imageFormat = parameters.getPreviewFormat();

        mCamera.setParameters(parameters);

        mCamera.startPreview();

        mCamera.setPreviewCallback(new Camera.PreviewCallback() {

            public void onPreviewFrame(final byte[] data, final Camera camera) {

                synchronized (this) {
                    totalFrames++;

                    if (imageFormat == ImageFormat.NV21) {

                        //We only accept the NV21(YUV420) format.
                        if (!bProcessing) {

                            totalProcFrames++;

                            matFrameCamera.put(0, 0, data);
                            mRgba2Gray = new Mat();
                            Imgproc.cvtColor(matFrameCamera, mRgba2Gray, Imgproc.COLOR_YUV420sp2GRAY);

                            mHandler.post(doImageProcessing);

                        }
                    }

                    this.notify();
                }

            }
        });
    }

    @Override
    public void surfaceCreated(SurfaceHolder arg0) {
        mCamera = Camera.open();
        try {
            // If did not set the SurfaceHolder, the preview area will be black.
            mCamera.setPreviewDisplay(arg0);
            mCamera.setDisplayOrientation(90);
            mCamera.setPreviewCallback(this);
        } catch (IOException e) {
            mCamera.release();
            mCamera = null;
        }
    }

    @Override
    public void surfaceDestroyed(SurfaceHolder arg0) {
        mCamera.setPreviewCallback(null);
        mCamera.stopPreview();
        mCamera.release();
        mRgba2Gray.release();
        matFrameCamera.release();
        matFrameOutputCamera.release();
        matTarget.release();
        mCamera = null;
    }

    //
    // Native JNI
    //
    public native void SurfMatcher(long matAddrFrameCamera, long matAddrFeature,
                                      long matAddrMatch);

    static {
        System.loadLibrary("native");
    }

    private Runnable doImageProcessing = new Runnable() {
        public void run() {

            if(!stop) {
                bProcessing = true;

                int bitmapWidth =  mRgba2Gray.cols() + matTarget.cols();
                int bitmapHeight = mRgba2Gray.rows() > matTarget.rows() ? mRgba2Gray.rows() : matTarget.rows();

                Mat matImageMatcher = new Mat(mRgba2Gray.rows(), bitmapWidth, CvType.CV_8UC1);

                SurfMatcher(mRgba2Gray.getNativeObjAddr(), matTarget.getNativeObjAddr(),
                            matImageMatcher.getNativeObjAddr());

                bitmap = Bitmap.createBitmap(bitmapWidth, bitmapHeight,
                        Bitmap.Config.ARGB_8888);
                Utils.matToBitmap(matImageMatcher, bitmap);

                ivCameraPreview.setImageBitmap(bitmap);

                stop = true;

                bProcessing = false;

            }

        }

    };


}

By making the image match the result is coming distorted (left image is camera frame and right is image target). I made the following test to show only the camera preview (mRgba2Gray variable) and this ok.

Answer 1

I was changing the data structure received by the camera and Mat result and operations.

@Override
    public void surfaceChanged(SurfaceHolder arg0, int arg1, int arg2, int arg3) {

        Camera.Parameters parameters = mCamera.getParameters();

        Camera.Size size = parameters.getPreviewSize();
        previewSizeHeight = size.height;
        previewSizeWidth = size.width;

        // Set the camera preview size
        parameters.setPreviewSize(previewSizeWidth, previewSizeHeight);

        imageFormat = parameters.getPreviewFormat();

        mCamera.setParameters(parameters);

        mCamera.startPreview();

        mCamera.setPreviewCallback(new Camera.PreviewCallback() {

            public void onPreviewFrame(final byte[] data, final Camera camera) {

                synchronized (this) {
                    totalFrames++;

                    if (imageFormat == ImageFormat.NV21) {

                        //We only accept the NV21(YUV420) format.
                        if (!bProcessing) {

                            if(matFrameCamera != null) {
                                matFrameCamera.release();
                            }

                            matFrameCamera = new Mat(previewSizeHeight + (previewSizeHeight / 2), previewSizeWidth, CvType.CV_8UC1);

                            totalProcFrames++;

                            matFrameCamera.put(0, 0, data);
                            mRgba2Gray = new Mat();
                            Imgproc.cvtColor(matFrameCamera, mRgba2Gray, Imgproc.COLOR_YUV2RGBA_NV21, 4);
                            mHandler.post(doImageProcessing);

                        }
                    }

                    this.notify();
                }

            }
        });
    }

    private Runnable doImageProcessing = new Runnable() {
        public void run() {

            if (!stop) {
                bProcessing = true;

                Imgproc.resize(mRgba2Gray, mRgba2Gray, new Size(mRgba2Gray.cols() / 3, mRgba2Gray.rows() / 3));
                Bitmap bitmapCameraPortrait = Bitmap.createBitmap(mRgba2Gray.cols(), mRgba2Gray.rows(),
                        Bitmap.Config.ARGB_8888);
                Utils.matToBitmap(mRgba2Gray, bitmapCameraPortrait);
                bitmap = scaleDown(bitmapCameraPortrait, mRgba2Gray.height() / 3, true);
                Utils.bitmapToMat(bitmapCameraPortrait, mRgba2Gray);
                Imgproc.cvtColor(mRgba2Gray, mRgba2Gray, Imgproc.COLOR_BGRA2GRAY);

                Mat matImageMatcher = new Mat();

                SurfMatcher(mRgba2Gray.getNativeObjAddr(), matTarget.getNativeObjAddr(),
                        matImageMatcher.getNativeObjAddr());

                int newWidthBitmap = bitmapCameraPortrait.getWidth() +
                        (matImageMatcher.cols() - bitmapCameraPortrait.getWidth());
                int newHeightBitmap = bitmapCameraPortrait.getHeight() +
                        (matImageMatcher.rows() - bitmapCameraPortrait.getHeight());

                bitmapCameraPortrait = Bitmap.createScaledBitmap(bitmapCameraPortrait, newWidthBitmap,
                        newHeightBitmap, true);

                Utils.matToBitmap(matImageMatcher, bitmapCameraPortrait);

                bitmap = bitmapCameraPortrait;

                ((Activity) context).runOnUiThread(new TimerTask() {
                    @Override
                    public void run() {
                        ivCameraPreview.setImageBitmap(bitmap);
                    }
                });

                stop = true;

                bProcessing = false;

            }

        }

    };