I want to play static HLS content (not live video content) on my app in Android. What I currently do is download all the segments from the .m3u8 file and merge it into one file. When I play this file, I can see this the video being played, but it is not seekable. As per this link, .ts files are not seekable on Android.
I cannot risk running ffmpeg on phone for converting the file to MP4 format. I have studied MP4 format and its atom structure. What I want to know is, if there is an easy way to create MP4 container (atoms hierarchy) which would simply refer to the .ts segment (the merged segment that was created from sub-segments) in its data atom (mdat)
I would really appreciate any help/suggestions.
Android provides support libraries such as MediaCodec and MediaExtractor that provides access to low level media encoding/decoding. It is fast and efficient as it uses hardware acceleration.
Here's how I believe one is suppose to do it on Android unless you are ok with using ffmpeg which of course if resource intensive operation.
1) Use MediaExtractor to extract data from the file.
2) Pass the extracted data to MediaCodec.
3) Use MediaCodec to render output to a surface (in case of video) and AudioTrack (in case of audio).
4) This is the most difficult step: Synchronize audio/video. I haven't implemented this yet. But this would require keeping track of time sync between audio and video. Audio would be played normally and you might have to drop some frames in case of video to keep it in sync with audio playback.
Here's code for decoding audio/video and playing them respectively using AudioTrack and Surface. In case of video decoding, there's a sleep to slowdown the frame rendering.
public void decodeVideo(Surface surface) throws IOException {
MediaExtractor extractor = new MediaExtractor();
MediaCodec codec;
ByteBuffer[] codecInputBuffers;
ByteBuffer[] codecOutputBuffers;
extractor.setDataSource(file);
Log.d(TAG, "No of tracks = " + extractor.getTrackCount());
MediaFormat format = extractor.getTrackFormat(0);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "mime = " + mime);
Log.d(TAG, "format = " + format);
codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, surface, null, 0);
codec.start();
codecInputBuffers = codec.getInputBuffers();
codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
final long timeout_in_Us = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
long startMs = System.currentTimeMillis();
while(!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if(!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(timeout_in_Us);
if(inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if(sampleSize < 0) {
Log.d(TAG, "saw input EOS.");
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if(!sawInputEOS) {
extractor.advance();
}
}
}
int res = codec.dequeueOutputBuffer(info, timeout_in_Us);
if(res >= 0) {
if(info.size > 0) {
noOutputCounter = 0;
}
int outputBufIndex = res;
while(info.presentationTimeUs/1000 > System.currentTimeMillis() - startMs) {
try {
Thread.sleep(5);
} catch (Exception e) {
break;
}
}
codec.releaseOutputBuffer(outputBufIndex, true);
if((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if(res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if(res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat format1 = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + format1);
} else if(res == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "Codec try again returned" + res);
}
}
codec.stop();
codec.release();
}
private int audioSessionId = -1;
private AudioTrack createAudioTrack(MediaFormat format) {
int channelConfiguration = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO;
int bufferSize = AudioTrack.getMinBufferSize(format.getInteger(MediaFormat.KEY_SAMPLE_RATE), channelConfiguration, AudioFormat.ENCODING_PCM_16BIT) * 8;
AudioTrack audioTrack;
if(audioSessionId == -1) {
audioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, format.getInteger(MediaFormat.KEY_SAMPLE_RATE), channelConfiguration,
AudioFormat.ENCODING_PCM_16BIT, bufferSize, AudioTrack.MODE_STREAM);
} else {
audioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, format.getInteger(MediaFormat.KEY_SAMPLE_RATE), channelConfiguration,
AudioFormat.ENCODING_PCM_16BIT, bufferSize, AudioTrack.MODE_STREAM, audioSessionId);
}
audioTrack.play();
audioSessionId = audioTrack.getAudioSessionId();
return audioTrack;
}
public void decodeAudio() throws IOException {
MediaExtractor extractor = new MediaExtractor();
MediaCodec codec;
ByteBuffer[] codecInputBuffers;
ByteBuffer[] codecOutputBuffers;
extractor.setDataSource(file);
Log.d(TAG, "No of tracks = " + extractor.getTrackCount());
MediaFormat format = extractor.getTrackFormat(1);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "mime = " + mime);
Log.d(TAG, "format = " + format);
codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null, null, 0);
codec.start();
codecInputBuffers = codec.getInputBuffers();
codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(1);
AudioTrack audioTrack = createAudioTrack(format);
final long timeout_in_Us = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while(!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if(!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(timeout_in_Us);
if(inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if(sampleSize < 0) {
Log.d(TAG, "saw input EOS.");
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if(!sawInputEOS) {
extractor.advance();
}
}
}
int res = codec.dequeueOutputBuffer(info, timeout_in_Us);
if(res >= 0) {
if(info.size > 0) {
noOutputCounter = 0;
}
int outputBufIndex = res;
//Possibly store the decoded buffer
ByteBuffer buf = codecOutputBuffers[outputBufIndex];
final byte[] chunk = new byte[info.size];
buf.get(chunk);
buf.clear();
if(chunk.length > 0) {
audioTrack.write(chunk, 0 ,chunk.length);
}
codec.releaseOutputBuffer(outputBufIndex, false);
if((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if(res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if(res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat format1 = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + format1);
audioTrack.stop();
audioTrack = createAudioTrack(codec.getOutputFormat());
} else if(res == MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.d(TAG, "Codec try again returned" + res);
}
}
codec.stop();
codec.release();
}