Audio is not included while using Android Mutex to generate video

Question

User Scenario: I want to export video clip of 15 seconds with sound from an ordinal video

Progress: I use Android Muxer https://developer.android.com/reference/android/media/MediaMuxer

I cannot include my audio even audio track is fetched

below is my code:

private fun splitVideo(inputPath: String, outputPath: String, startTimeMs: Long, endTimeMs: Long) {

    val outputFile = File(outputPath)
    if (outputFile.exists()) {
        outputFile.delete()
    }

    val mediaExtractor = MediaExtractor()
    val mediaMuxer = MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
    try {
        mediaExtractor.setDataSource(inputPath)

        var newVideoWidth = 0
        var newVideoHeight = 0

        val inputVideoRetriever = MediaMetadataRetriever()
        inputVideoRetriever.setDataSource(inputPath)

        //fetch orientation
        val rotation = inputVideoRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)
        if (rotation == null || rotation == MEDIA_ORIENTAITON_0 || rotation == MEDIA_ORIENTAITON_180) {
            newVideoWidth = inputVideoRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toIntOrNull() ?: 0
            newVideoHeight = inputVideoRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toIntOrNull() ?: 0
        } else {
            // if it is rotated to it's side, swap width and height from metadata
            newVideoWidth = inputVideoRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toIntOrNull() ?: 0
            newVideoHeight = inputVideoRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toIntOrNull() ?: 0
        }
        val orientationHint = when (rotation) {
            MEDIA_ORIENTAITON_90 -> 90
            MEDIA_ORIENTAITON_270 -> 270
            else -> 0 // Default to 0 for no rotation or 180-degree rotation
        }

        mediaMuxer.setOrientationHint(orientationHint)

        val videoTrackIndex = selectTrack(mediaExtractor, "video/")
        val audioTrackIndex = selectTrack(mediaExtractor, "audio/")
        println("#20230807X videoTrackIndex: $videoTrackIndex")
        println("#20230807X audioTrackIndex: $audioTrackIndex")

        var startTimeUs = startTimeMs * 1000
        val endTimeUs = endTimeMs * 1000

        val buffer = ByteBuffer.allocate(2 * 1024 * 1024)
        val bufferInfo = MediaCodec.BufferInfo()

        if (videoTrackIndex >= 0) {
            mediaExtractor.selectTrack(videoTrackIndex)
            mediaExtractor.selectTrack(audioTrackIndex)

            val videoMediaFormat = mediaExtractor.getTrackFormat(videoTrackIndex)
            if (newVideoWidth > 0 && newVideoHeight > 0) {
                videoMediaFormat.setInteger(MediaFormat.KEY_WIDTH, newVideoWidth)
                videoMediaFormat.setInteger(MediaFormat.KEY_HEIGHT, newVideoHeight)
                videoMediaFormat.setLong(MediaFormat.KEY_DURATION, 15000L)
            }

            //begin sampling to split video
            val videoMuxerTrackIndex = mediaMuxer.addTrack(videoMediaFormat)

            val audioFormat = mediaExtractor.getTrackFormat(audioTrackIndex)
            val audioMuxerTrackIndex = mediaMuxer.addTrack(audioFormat)

            mediaMuxer.start()

            mediaExtractor.seekTo(startTimeUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)

            while (true) {
                val sampleSize = mediaExtractor.readSampleData(buffer, 0)

                if (sampleSize < 0 || mediaExtractor.sampleTime > endTimeUs) {
                    break
                }

                bufferInfo.presentationTimeUs = mediaExtractor.sampleTime
                bufferInfo.offset = 0
                bufferInfo.size = sampleSize
                bufferInfo.flags = MediaCodec.BUFFER_FLAG_KEY_FRAME

                mediaMuxer.writeSampleData(videoMuxerTrackIndex, buffer, bufferInfo)
                mediaMuxer.writeSampleData(audioMuxerTrackIndex, buffer, bufferInfo)
                mediaExtractor.advance()
            }
        }

        mediaMuxer.stop()

    

    } catch (e: Exception) {
        /*
        Make sure the following permissions are granted:
        <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
        <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
         */
        e.printStackTrace()
    } finally {
        mediaExtractor.release()
        mediaMuxer.release()
    }
}

Answer 1

Try to create two extractors, and read/write tracks in two loops:

    while (true) {
            val sampleSize = videoExtractor.readSampleData(videoBuffer, 0)
            // ...
            mediaMuxer.writeSampleData(videoMuxerTrackIndex, videoBuffer, videoBufferInfo)
            videoExtractor.advance()
    }

    while (true) {
            val sampleSize = audioExtractor.readSampleData(audioBuffer, 0)
            // ...
            mediaMuxer.writeSampleData(audioMuxerTrackIndex, audioBuffer, audioBufferInfo)
            audioExtractor.advance()
    }