I want to run a image classifier in a infinite loop on a background thread. The function should be called immediately after launching the app. I want to feed the classifier with current frames from a prerecorded video which is simultaneously playing in the UI-thread, so the background thread should tell the UI-thread, once it's done, so I can feed it with the current frame and rerun the classifier.
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
private VideoView videoView;
private ImageView imageView;
private Uri uri_video;
private MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
private MediaController mMediaController;
private static volatile int currentPosition;
private static volatile Bitmap mBitmap;
private final Object lock = new Object();
private volatile boolean runClassifier = false;
private HandlerThread backgroundThread;
private Handler backgroundHandler;
private static final String HANDLE_THREAD_NAME = "ClassifierBackground";
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
imageView = findViewById(R.id.imageView);
uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.kim);
mediaMetadataRetriever = new MediaMetadataRetriever();
mediaMetadataRetriever.setDataSource(getApplication(), uri_video);
videoView = findViewById(R.id.videoView);
videoView.setVideoURI(uri_video);
mMediaController = new MediaController(this);
videoView.setMediaController(mMediaController);
videoView.setOnPreparedListener(MyVideoViewPreparedListener);
videoView.start();
startBackgroundThread();
}
/** Starts a background thread and its {@link Handler}. */
private void startBackgroundThread() {
backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
backgroundThread.start();
backgroundHandler = new Handler(backgroundThread.getLooper());
synchronized (lock) {
runClassifier = true;
}
backgroundHandler.post(periodicClassify);
}
/** Stops the background thread and its {@link Handler}. */
private void stopBackgroundThread() {
backgroundThread.quitSafely();
try {
backgroundThread.join();
backgroundThread = null;
backgroundHandler = null;
synchronized (lock) {
runClassifier = false;
}
} catch (InterruptedException e) {
Log.e(TAG, "Interrupted when stopping background thread", e);
}
}
private Runnable periodicClassify =
new Runnable() {
@Override
public void run() {
synchronized (lock) {
if (runClassifier) {
// classifyFrame(); // This will be implemented later
Log.d(TAG, "run: Classifier is running");
SystemClock.sleep(100); // Instead I simulate the classifier via sleep
}
}
runOnUiThread(new Runnable() {
@Override
public void run() {
setImageViewToCurrentFrame();
}
});
backgroundHandler.post(periodicClassify);
}
};
private void setImageViewToCurrentFrame(){
currentPosition = videoView.getCurrentPosition(); //in millisecond
mBitmap = mediaMetadataRetriever
.getFrameAtTime(currentPosition * 1000); //unit in microsecond
imageView.setImageBitmap(mBitmap);
}
MediaPlayer.OnPreparedListener MyVideoViewPreparedListener =
new MediaPlayer.OnPreparedListener() {
@Override
public void onPrepared(MediaPlayer mp) {
long duration = videoView.getDuration(); //in millisecond
Toast.makeText(MainActivity.this,
"Duration: " + duration + " (ms)",
Toast.LENGTH_LONG).show();
setImageViewToCurrentFrame();
}
};
@Override
protected void onDestroy() {
super.onDestroy();
stopBackgroundThread();
}
EDIT1:
I got some rough idea on how to do it from these videos. It seems like I need a backgroundThread (HandlerThread) that has a backgroundHandler (Handler) to communicate with the UI-thread and a Looper to keep the background thread alive. setImageViewToCurrentFrame uses videoView.getCurrentPosition() to update the mBitmap.
However the update is very slow (>10 seconds) compared to the runtime of the classifier (SystemClock.sleep(100) which takes 100ms).
EDIT2: The problem seems to be the performance of ImageView which seems to be updated very slowly. Replacing it with TextView, keeps both the background thread and the UI-thread in sync. I will look for other solutions than ImageView now
Here is the gist of my solution. One needs to fit in the actual image classifier at the SystemClock.sleep part.
The trick is to use TextureView instead of ImageView or VideoView, since it's faster and more flexible
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.video_playback);
initializeBottomSheet();
uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.my_demo_video);
textureView = findViewById(R.id.textureView);
textureView.setSurfaceTextureListener(this);
mediaPlayer = new MediaPlayer();
assert textureView != null;
startBackgroundThread();
}
/** Starts a background thread and its {@link Handler}. */
private void startBackgroundThread() {
backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
backgroundThread.start();
backgroundHandler = new Handler(backgroundThread.getLooper());
synchronized (lock) {
runClassifier = true;
}
backgroundHandler.post(periodicClassify);
}
/** Stops the background thread and its {@link Handler}. */
private void stopBackgroundThread() {
backgroundThread.quitSafely();
try {
backgroundThread.join();
backgroundThread = null;
backgroundHandler = null;
synchronized (lock) {
runClassifier = false;
}
} catch (InterruptedException e) {
Log.e(TAG, "Interrupted when stopping background thread", e);
}
}
private Runnable periodicClassify =
new Runnable() {
@Override
public void run() {
// Get current frame from video playback
mBitmap = textureView.getBitmap();
if (classifier != null && mBitmap != null) {
Log.d(TAG, "Classifier: Start thread");
SystemClock.sleep(3000); // Instead I simulate the classifier via sleep
}
backgroundHandler.post(periodicClassify);
}
};
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
Surface surface = new Surface(surfaceTexture);
Context context = getApplicationContext();
try {
mediaPlayer.setDataSource(context, uri_video);
mediaPlayer.setSurface(surface);
mediaPlayer.prepareAsync();
mediaPlayer.setOnPreparedListener(new MediaPlayer.OnPreparedListener(){
@Override
public void onPrepared(MediaPlayer mediaPlayer) {
mediaPlayer.start();
textureView.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View view) {
if (mediaPlayer.isPlaying()){
mediaPlayer.pause();
}else{
mediaPlayer.start();
}
}
});
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
protected void onDestroy() {
super.onDestroy();
stopBackgroundThread();
if(mediaPlayer != null){
mediaPlayer.stop();
mediaPlayer.release();
mediaPlayer = null;
}
}
@Override
protected void onPause() {
super.onPause();
if(mediaPlayer != null && mediaPlayer.isPlaying()){
mediaPlayer.pause();
}
}
@Override
protected void onResume() {
super.onResume();
if(mediaPlayer != null && mediaPlayer.isPlaying()){
mediaPlayer.start();
}
startBackgroundThread();
}