java android opencv camera android-camera

OpenCV - How to set a Full Screen Camera view in Android?

Background

As a novice in software, I am currently aiming to format my camerapreview to full screen, that would be identical to the camera preview on Snapchat. Right now, I am able to showcase my camera preview in a 1:1 box format that I was able to set by following this tutorial. Other potential solutions that I had encountered in other questions either stretched/distorted the previewed image, or didn't launch the application altogether. How would I be able to do this while still maintaining portrait mode? Code provided below

Additional device specifications include the fact that the device I aim to launch the application on is a OnePlus Six, and its aspect ratio is 19:9. This is what the camera on my application currently looks like this.

I want to eliminate the black borders above and below the preview and allow the camera to take up the entirety of the screen.

MainActivity.java

package com.example.cv;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;

import android.Manifest;
import android.content.pm.PackageManager;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceView;
import android.view.WindowManager;
import android.widget.Toast;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.imgproc.Imgproc;

public class MainActivity extends AppCompatActivity implements CameraBridgeViewBase.CvCameraViewListener2
{
    private static String TAG = "MainActivity";
    JavaCameraView javaCameraView;
    Mat mRGBA, mRGBAT, dst;

    private static final int MY_CAMERA_REQUEST_CODE = 100;


    BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(MainActivity.this) {
        @Override
        public void onManagerConnected(int status)
        {
            if (status == BaseLoaderCallback.SUCCESS) {
                javaCameraView.enableView();
            } else {
                super.onManagerConnected(status);
            }
        }
    };

    static
    {
        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
        }
    }

    @Override
    protected void onCreate(Bundle savedInstanceState)
    {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        javaCameraView = (JavaCameraView) findViewById(R.id.my_camera_view);



        if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
                == PackageManager.PERMISSION_GRANTED)  {
            Log.d(TAG, "Permissions granted");
            javaCameraView.setCameraPermissionGranted();
            javaCameraView.setCameraIndex(CameraBridgeViewBase.CAMERA_ID_BACK);
            javaCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
            javaCameraView.setCvCameraViewListener(this);
        } else {
            Log.d(TAG, "Permission prompt");
            ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, MY_CAMERA_REQUEST_CODE);
        }



    }

    @Override
    public void onCameraViewStarted(int width, int height)
    {
        mRGBAT = new Mat();
        dst = new Mat();
    }

    @Override
    public void onCameraViewStopped()
    {
        mRGBA.release();
    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame)
    {
        mRGBA = inputFrame.rgba();
        Core.transpose(mRGBA, mRGBAT);
        Core.flip(mRGBAT, mRGBAT, 1);
        Imgproc.resize(mRGBAT, dst, mRGBA.size());
        mRGBA.release();
        mRGBAT.release();
        return dst;
    }

    @Override
    public void onPointerCaptureChanged(boolean hasCapture) {

    }


    @Override
    protected void onDestroy() {
        super.onDestroy();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }

    @Override
    protected void onPause() {
        super.onPause();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }


    @Override
    protected void onResume() {
        super.onResume();

        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
            baseLoaderCallback.onManagerConnected(BaseLoaderCallback.SUCCESS);
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
            OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
        }
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
        if (requestCode == MY_CAMERA_REQUEST_CODE) {
            // camera can be turned on
            Toast.makeText(this, "camera permission granted", Toast.LENGTH_LONG).show();
            javaCameraView.setCameraPermissionGranted();
            javaCameraView.setCameraIndex(CameraBridgeViewBase.CAMERA_ID_FRONT);
            javaCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
            javaCameraView.setCvCameraViewListener(this);
        } else {
            //camera will stay off
            Toast.makeText(this, "camera permission denied", Toast.LENGTH_LONG).show();
        }
    }
}

activity_main.xml

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout
    xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".MainActivity">


    <org.opencv.android.JavaCameraView
        android:id="@+id/my_camera_view"
        android:layout_width="fill_parent"
        android:layout_height="fill_parent"
        />

</RelativeLayout>

AndroidManifest.xml

<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
    package="com.example.cv">

    <supports-screens android:resizeable="true"
        android:smallScreens="true"
        android:normalScreens="true"
        android:largeScreens="true"
        android:anyDensity="true" />


    <uses-permission android:name="android.permission.CAMERA"/>
    <uses-feature android:name="android.hardware.camera"/>
    <uses-feature android:name="android.hardware.camera.autofocus"/>
    <uses-feature android:name="android.hardware.camera.front"/>
    <uses-feature android:name="android.hardware.camera.front.autofocus"/>

    <application
        android:allowBackup="true"
        android:icon="@mipmap/ic_launcher"
        android:label="@string/app_name"
        android:roundIcon="@mipmap/ic_launcher_round"
        android:supportsRtl="true"
        android:theme="@style/Theme.AppCompat.Light.NoActionBar">
        <activity android:name=".MainActivity"
            android:screenOrientation="portrait"
            android:configChanges="keyboardHidden|orientation">
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />

                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
    </application>

</manifest>

UPDATE: I have come across solutions that aim to change orientation and/or eliminate the action bar. Applying setMaxFrameSize() does not work aside from stretching the resolution of the CameraPreview. Another answer I have seen is getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); which is no longer valid as FLAG_KEEP_SCREEN_ON has been deprecated. If anyone can offer the slightest bit of solution as to how I can fix this, I would be eternally grateful.

UPDATE 2: I have attempted to modify layout_height in my activity_main.xml file, only to have it push the preview further down the screen, while it still retains it's 1:1 box format. Additionally, I have also considered implementing javaCameraView.getLayoutParams().height= in my MainActivity only to have it distort/stretch the camera preview and not achieve my intended desire.

Solution

Short answer

I'm leaving at the end the full solution for the MainActivity class - both for the back and front cameras - after the code explanations

Full Details

I joined a few answers on this website and modified onCameraFrame(). Now I can open the app in portrait mode, as the native Android camera or Snapchat view

This solution is really lightweight, I didn't change any OpenCV file as in other answers on this website [e.g. this one]. Depending on the activeCamera you want to use [i.e. back or front one] there is a little modification required inside onCameraFrame(), which I will list below

Common base

So I first followed the 4 steps in this answer, exactly as they are presented there and they really put me on the right track. It is worth to notice that with these changes you will get immediately a good looking landscape view but as soon as you put your phone in a portrait position the upper and lower black bands will come back. Changing the line at step 3 into:

android:screenOrientation="portrait"

didn't solve for me. Also, if you want to go full screen you need to remove the title bar and in order to do so I incorporated this answer too. This means that you will need to modify also this line in the AndroidManifest.xml from the previous step 2

android:theme="@style/Theme.AppCompat.Light.NoActionBar.FullScreen"

Since you initialize the camera in two places and you need to change it [the current snippet works with the front camera, you need the back one] you can clean-up the code and start defining the camera of interest so that you can modify the input source in one point only:

public class MainActivity extends AppCompatActivity implements
        CameraBridgeViewBase.CvCameraViewListener2
{
    ...
    // back camera
    int activeCamera = CameraBridgeViewBase.CAMERA_ID_BACK;
    // front camera
    // int activeCamera = CameraBridgeViewBase.CAMERA_ID_FRONT;

then pass it to a new initializeCamera() method

private void initializeCamera(JavaCameraView javaCameraView, int activeCamera){
    javaCameraView.setCameraPermissionGranted();
    javaCameraView.setCameraIndex(activeCamera);
    javaCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
    javaCameraView.setCvCameraViewListener(this);
}

which you have to call as soon as Android detects that the user has given the CAMERA permission:

    if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
            == PackageManager.PERMISSION_GRANTED) {
        Log.d(TAG, "Permissions granted");
        initializeCamera(javaCameraView, activeCamera);

and:

    if (requestCode == MY_CAMERA_REQUEST_CODE) {
        if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
            Toast.makeText(this, "Camera Permission granted", Toast.LENGTH_LONG).show();
            initializeCamera(javaCameraView, activeCamera);

Now it's the moment to distinguish between front and back camera

Back camera

In this case you won't need to manipulate the frames at all and the code goes as follows:

@Override
public void onCameraViewStarted(int width, int height){
}

@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame)
{
    mRGBA = inputFrame.rgba();
    return mRGBA;
}

Front camera

In this case you need to slightly manipulate the frames flipping them, otherwise your portrait mode will show upside down

@Override
public void onCameraViewStarted(int width, int height)
{
    mRGBAT = new Mat();
} 

@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame)
{
    mRGBA = inputFrame.rgba();
    // flipping to show portrait mode properly
    Core.flip(mRGBA, mRGBAT, 1);
    // releasing what's not anymore needed
    mRGBA.release();
    return mRGBAT;
}

Finally, be sure that the frame manipulations you add to your code are really required. Their processing time lowers performances and can even expose you to unrequired troubles. I didn't check accurately but I'm quite sure that transposing the matrices in your onCameraFrame() was the root cause for distortion

MainActivity for back camera

package com.change.package.name;

import android.Manifest;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.util.Log;
import android.view.WindowManager;
import android.widget.Toast;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Mat;

public class MainActivity extends AppCompatActivity implements
        CameraBridgeViewBase.CvCameraViewListener2
{
    private static String TAG = "MainActivity";
    JavaCameraView javaCameraView;
    Mat mRGBA;
    private static final int MY_CAMERA_REQUEST_CODE = 100;
    int activeCamera = CameraBridgeViewBase.CAMERA_ID_BACK;


    BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(MainActivity.this) {
        @Override
        public void onManagerConnected(int status)
        {
            if (status == BaseLoaderCallback.SUCCESS) {
                javaCameraView.enableView();
            } else {
                super.onManagerConnected(status);
            }
        }
    };

    static
    {
        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
        }
    }

    @Override
    protected void onCreate(Bundle savedInstanceState)
    {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        javaCameraView = (JavaCameraView) findViewById(R.id.my_camera_view);
        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

        if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
                == PackageManager.PERMISSION_GRANTED) {
            Log.d(TAG, "Permissions granted");
            initializeCamera(javaCameraView, activeCamera);
        } else {
            Log.d(TAG, "Troubles");
            ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, MY_CAMERA_REQUEST_CODE);
            }
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
        if (requestCode == MY_CAMERA_REQUEST_CODE) {
            if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
                Toast.makeText(this, "Camera Permission granted", Toast.LENGTH_LONG).show();  
                initializeCamera(javaCameraView, activeCamera);
            } else {
                Toast.makeText(this, "Camera Permission denied", Toast.LENGTH_LONG).show();
            }
        }
    }

    private void initializeCamera(JavaCameraView javaCameraView, int activeCamera){
        javaCameraView.setCameraPermissionGranted();
        javaCameraView.setCameraIndex(activeCamera);

        javaCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
        javaCameraView.setCvCameraViewListener(this);
    }

    @Override
    public void onCameraViewStarted(int width, int height)
    {
    
    }

    @Override
    public void onCameraViewStopped()
    {
        mRGBA.release();
    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame)
    {
        // code for the back camera
        mRGBA = inputFrame.rgba();
        return mRGBA;
    }

    @Override
    public void onPointerCaptureChanged(boolean hasCapture) {

    }    

    @Override
    protected void onDestroy() {
        super.onDestroy();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }

    @Override
    protected void onPause() {
        super.onPause();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }    

    @Override
    protected void onResume() {
        super.onResume();

        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
            baseLoaderCallback.onManagerConnected(BaseLoaderCallback.SUCCESS);
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
            OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
        }
    }
}

MainActivity for front camera

package com.change.package.name;

import android.Manifest;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.util.Log;
import android.view.WindowManager;
import android.widget.Toast;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;

import org.opencv.android.BaseLoaderCallback;
import org.opencv.android.CameraBridgeViewBase;
import org.opencv.android.JavaCameraView;
import org.opencv.android.OpenCVLoader;
import org.opencv.core.Core;
import org.opencv.core.Mat;

public class MainActivity extends AppCompatActivity implements
        CameraBridgeViewBase.CvCameraViewListener2
{
    private static String TAG = "MainActivity";
    JavaCameraView javaCameraView;
    Mat mRGBA, mRGBAT;
    private static final int MY_CAMERA_REQUEST_CODE = 100;
    int activeCamera = CameraBridgeViewBase.CAMERA_ID_FRONT;


    BaseLoaderCallback baseLoaderCallback = new BaseLoaderCallback(MainActivity.this) {
        @Override
        public void onManagerConnected(int status)
        {
            if (status == BaseLoaderCallback.SUCCESS) {
                javaCameraView.enableView();
            } else {
                super.onManagerConnected(status);
            }
        }
    };

    static
    {
        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
        }
    }

    @Override
    protected void onCreate(Bundle savedInstanceState)
    {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        javaCameraView = (JavaCameraView) findViewById(R.id.my_camera_view);
        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

        if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
                == PackageManager.PERMISSION_GRANTED) {
            Log.d(TAG, "Permissions granted");
            initializeCamera(javaCameraView, activeCamera);
        } else {
            Log.d(TAG, "Troubles");
            ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, MY_CAMERA_REQUEST_CODE);
            }
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
        if (requestCode == MY_CAMERA_REQUEST_CODE) {
            if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
                Toast.makeText(this, "Camera Permission granted", Toast.LENGTH_LONG).show();
                initializeCamera(javaCameraView, activeCamera);
            } else {
                Toast.makeText(this, "Camera Permission denied", Toast.LENGTH_LONG).show();
            }
        }
    }

    private void initializeCamera(JavaCameraView javaCameraView, int activeCamera){
        javaCameraView.setCameraPermissionGranted();
        javaCameraView.setCameraIndex(activeCamera);   
        javaCameraView.setVisibility(CameraBridgeViewBase.VISIBLE);
        javaCameraView.setCvCameraViewListener(this);
    }

    @Override
    public void onCameraViewStarted(int width, int height)
    {
        mRGBAT = new Mat();
    }

    @Override
    public void onCameraViewStopped()
    {
        mRGBA.release();
    }

    @Override
    public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame)
    {
        // code for the front camera
        mRGBA = inputFrame.rgba();
        // flipping to show portrait mode properly
        Core.flip(mRGBA, mRGBAT, 1);
        mRGBA.release();
        return mRGBAT;
    }

    @Override
    public void onPointerCaptureChanged(boolean hasCapture) {

    }    

    @Override
    protected void onDestroy() {
        super.onDestroy();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }

    @Override
    protected void onPause() {
        super.onPause();

        if (javaCameraView != null)
        {
            javaCameraView.disableView();
        }
    }    

    @Override
    protected void onResume() {
        super.onResume();

        if (OpenCVLoader.initDebug())
        {
            Log.d(TAG, "OpenCV is Configured or Connected successfully.");
            baseLoaderCallback.onManagerConnected(BaseLoaderCallback.SUCCESS);
        }
        else
        {
            Log.d(TAG, "OpenCV not Working or Loaded.");
            OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION, this, baseLoaderCallback);
        }
    }
}