I have a legacy code written by someone else and I just recently realised that the video recorded in the app has different length when it is played outside the app or programmatically checked using MediaMetadataRetriever.METADATA_KEY_DURATION than the measured time using the frames' timeStamps.
The difference is almost 20% longer video than measured.
Below is the printout from the frame timestamps:
frameRate: 30
estimateStarttime: 83990376186060
estimateEndTime: 84009908074060
Which translates to 19.53 seconds while results from the MediaPlayer and MediaMetadataRetriever.METADATA_KEY_DURATION
duration player for Example-video is 23520
duration flag for Example-video is 23520
And here is a snippet of my code:
final ArrayList<Surface> als = new ArrayList<>();
final Surface is;
Range<Integer> targetFps = null;
Range<Integer>[] ranges = targetDetails.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES);
if(ranges != null)
for (Range<Integer> range : ranges) {
//Request the highest available lower==upper fps range.
Log.d("FPSRangeMeta", String.format(Locale.getDefault(), "(%dx%d): [%d..%d]", 1280, 720, range.getLower(), range.getUpper()));
if(range.getUpper().equals(range.getLower())){
if(targetFps == null)
targetFps = range;
if(range.getLower() > targetFps.getLower())
targetFps = range;
}
}
//noinspection ConstantConditions targetFps to be implemented
final int framerate = targetFps != null?targetFps.getLower():15;
System.out.println("frameRate: " + framerate);
if (recording) {
MediaCodec.Callback cb = new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec mc, int inputBufferId) {
throw new RuntimeException("Shouldn't be called with an input surface.");
}
@SuppressLint("LogConditional")
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) {
try {
ByteBuffer outputBuffer = Objects.requireNonNull(codec.getOutputBuffer(index));
//codec.getOutputFormat(index); // option A
synchronized (CameraHandler.this) {
long flags = info.flags;
if(estimateTimestamp == 0 && estimateStarttime != 0) {
estimateTimestamp = estimateStarttime;
System.out.println("estimateStarttime: " + estimateStarttime);
}
encodedFrameCallback.pushFrameMeta("frame.keyframe", frameIndex, flags & MediaCodec.BUFFER_FLAG_KEY_FRAME);
encodedFrameCallback.pushFrameMeta("frame.output_time_info", frameIndex, info.presentationTimeUs*1000);
encodedFrameCallback.pushFrameMeta("frame.frame_delta", frameIndex, info.presentationTimeUs*1000 - prev);
if(prev != 0){
estimateTimestamp += info.presentationTimeUs*1000 - prev;
}
encodedFrameCallback.pushFrameMeta("frame.estimated_timestamp", frameIndex, estimateTimestamp);
encodedFrameCallback.pushH264Frame(outputBuffer, info.offset, info.size, info.presentationTimeUs);
prev = info.presentationTimeUs*1000;
frameIndex++;
}
codec.releaseOutputBuffer(index, false);
} catch (IllegalStateException e) {
Timber.e(e);
}
}
@Override
public void onError(@NonNull MediaCodec codec, @NonNull MediaCodec.CodecException e) {
Log.d("MCError", "MediaCodec error");
e.printStackTrace();
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec mc, @NonNull MediaFormat format) {
}
};
//noinspection HardCodedStringLiteral
MediaFormat mf = MediaFormat.createVideoFormat("video/avc", 1280, 720);
Range<Integer> complexityRange = codec.getCodecInfo().getCapabilitiesForType("video/avc").getEncoderCapabilities().getComplexityRange();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
Range<Integer> qualityRange = codec.getCodecInfo().getCapabilitiesForType("video/avc").getEncoderCapabilities().getQualityRange();
Log.d("CameraHandler", "Available quality range: "+qualityRange.getLower()+" --- "+qualityRange.getUpper());
if(!qualityRange.getLower().equals(qualityRange.getUpper())) mf.setInteger(MediaFormat.KEY_QUALITY, qualityRange.getUpper());
} else {
Log.d("CameraHandler", "Can't set quality, Android version too low");
}
MediaCodecInfo.EncoderCapabilities ec = codec.getCodecInfo().getCapabilitiesForType("video/avc").getEncoderCapabilities();
Log.d("CameraHandler", "CBR:" + ec.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR));
Log.d("CameraHandler", "CQ:" + ec.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CQ));
Log.d("CameraHandler", "VBR:" + ec.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR));
Log.d("CameraHandler", "Available complexity range: "+complexityRange.getLower()+" --- "+complexityRange.getUpper());
mf.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
mf.setInteger(MediaFormat.KEY_BIT_RATE, 12140318);//Same as sample file from Xperia Z5 recording.
mf.setInteger(MediaFormat.KEY_FRAME_RATE, framerate);
mf.setInteger(MediaFormat.KEY_CAPTURE_RATE, framerate);
mf.setInteger(MediaFormat.KEY_PRIORITY, 0);
mf.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5);
mf.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 100000000);
mf.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
mf.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel41);
try {
codec.reset();
codec.setCallback(cb);
codec.configure(mf, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch(Exception failed_hi41){
mf.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
mf.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31);
failed_hi41.printStackTrace();
try {
codec.reset();
codec.setCallback(cb);
codec.configure(mf, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch(Exception failed_hi31){
mf.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileMain);
mf.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31);
codec.reset();
codec.setCallback(cb);
codec.configure(mf, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
}
}
is = codec.createInputSurface();
als.add(is);
}
boolean finalUseMotionTrack = useMotionTrack;
Range<Integer> finalTargetFps = targetFps;
CameraDevice.StateCallback cameraCallback = new CameraDevice.StateCallback() {
@Override
public void onOpened(@NonNull final CameraDevice camera) {
currentCamera = camera;
if(closed) return;
try {
if (dicb != null)
dicb.deviceInfo(
codec.getOutputFormat(),
camera,
framerate,
cameraManager.getCameraCharacteristics(camera.getId()));
} catch (CameraAccessException e) {
e.printStackTrace();
}
try {
final CaptureRequest.Builder cr = camera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
//Use control mode AUTO so we can access A3 features.
cr.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO);
//Use standard video-oriented auto focus - since some devices can't live without it...
cr.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_VIDEO);
//Use standard white balance optimization
cr.set(CaptureRequest.CONTROL_AWB_MODE, CaptureRequest.CONTROL_AWB_MODE_AUTO);
//Enable auto exposure. This is not ideal, but manual configuration is hard over multiple devices.
cr.set(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON);
if(finalTargetFps != null) cr.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, finalTargetFps);
//Request widest available field of view
float focalLengths[] = targetDetails.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS);
if(focalLengths != null && focalLengths.length > 0)
cr.set(CaptureRequest.LENS_FOCAL_LENGTH, focalLengths[0]);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && finalUseMotionTrack) {
cr.set(CaptureRequest.CONTROL_CAPTURE_INTENT, CaptureRequest.CONTROL_CAPTURE_INTENT_MOTION_TRACKING);
} else {
cr.set(CaptureRequest.CONTROL_CAPTURE_INTENT, CaptureRequest.CONTROL_CAPTURE_INTENT_VIDEO_RECORD);
}
cr.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_OFF);
cr.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_OFF);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
cr.set(CaptureRequest.STATISTICS_OIS_DATA_MODE, CaptureRequest.STATISTICS_OIS_DATA_MODE_OFF);
}
for (Surface al : als) {
cr.addTarget(al);
}
CameraCaptureSession.StateCallback sessionCallback = new CameraCaptureSession.StateCallback() {
@Override
public void onConfigured(@NonNull CameraCaptureSession session) {
if(closed) return;
if (recording) codec.start();
try {
CameraCaptureSession.CaptureCallback cc = new CameraCaptureSession.CaptureCallback() {
@Override
public void onCaptureCompleted(@NonNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result) {
super.onCaptureCompleted(session, request, result);
long frame = result.getFrameNumber();
if(estimateStarttime == 0){
Long ts = result.get(CaptureResult.SENSOR_TIMESTAMP);
estimateStarttime = ts!=null?ts:0;
}
Long timestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
Long exposuretime = result.get(CaptureResult.SENSOR_EXPOSURE_TIME);
Long shutterskew = result.get(CaptureResult.SENSOR_ROLLING_SHUTTER_SKEW);
Long frameduration = result.get(CaptureResult.SENSOR_FRAME_DURATION);
Long clock_timing_monotonic = System.nanoTime();
Long clock_timing_realtime = SystemClock.elapsedRealtimeNanos();
Long clock_timing_millis = System.currentTimeMillis();
if(encodedFrameCallback != null){
encodedFrameCallback.pushFrameMeta("comparable.realtime_time", frame, clock_timing_realtime);
encodedFrameCallback.pushFrameMeta("comparable.monotonic_time", frame, clock_timing_monotonic);
encodedFrameCallback.pushFrameMeta("comparable.millis_time", frame, clock_timing_millis);
if(timestamp != null) encodedFrameCallback.pushFrameMeta("sensor.timestamp", frame, timestamp);
if(exposuretime != null) encodedFrameCallback.pushFrameMeta("sensor.exposure_time", frame, exposuretime);
if(shutterskew != null) encodedFrameCallback.pushFrameMeta("sensor.rolling_shutter_skew", frame, shutterskew);
if(frameduration != null) encodedFrameCallback.pushFrameMeta("sensor.frame_duration", frame, frameduration);
}
}
};
if(closed) return;
Handler h = new Handler(captureLooperThread.getLooper());
//if(finalSc != null) finalSc.setCaptureCallback(cc, h);
session.setRepeatingRequest(cr.build(), cc, h);
} catch (CameraAccessException | IllegalStateException e) {
e.printStackTrace();
onFailure.run();
}
}
@Override
public void onConfigureFailed(@NonNull CameraCaptureSession session) {
Timber.e("onConfigureFailed", new Exception("Failed to configure camera??"));
onFailure.run();
}
};
camera.createCaptureSession(als, sessionCallback, new Handler(recordLooperThread.getLooper()));
} catch (CameraAccessException | IllegalStateException e) {
e.printStackTrace();
onFailure.run();
}
}
@Override
public void onDisconnected(@NonNull CameraDevice camera) { }
@Override
public void onError(@NonNull CameraDevice camera, int error) {
Timber.e("Camera error "+error, new Exception("Got a camera error, "+error));
}
};
cameraManager.openCamera(targetCamera, cameraCallback, new Handler(stateLooperThread.getLooper()));
return true;
} catch (CameraAccessException e) {
e.printStackTrace();
return false;
}
}
public void stopRecording(){
closed = true;
if(currentCamera != null) currentCamera.close();
currentCamera = null;
System.out.println("estimateEndTime: " + estimateTimestamp);
codec.stop();
codec.reset();
}
After digging deeper into my code I found that the problem is not with the MediaCode settings/configurations. But the problem was in storing the file as you can see below:
data.position(offset);
data.limit(offset + size);//This causes the stored video to be longer than it is in reality.
try {
while (data.remaining() > 0)
h264Stream.getChannel().write(data);
} catch (IOException e) {}
removing the line data.limit(offset + size); fixed my problem.
The offset is
The start-offset of the data in the buffer.
And the size is
The amount of data (in bytes) in the buffer. If this is 0, the buffer has no data in it and can be discarded. The only use of a 0-size buffer is to carry the end-of-stream marker.
I still don't understand the repercussions of removing that line but I am going to do extensive tests. Please drop comments or a better answer if you have knowledge about it. I will change the correct answer!