Search code examples
androidiphonevoiplinphoneopenh264

How to fix black screen when call is answered and loss packs on H264


I am developing VOIP app using linphone IOS/Android library.

Basically I used OpenH264 video codec.

When call is answered, both side saw black screen.

After long time, both size can see the video each other.

When I see the log, there are many packets loss so that First frame can not be decoded.

At LinphoneCallStreamsRunning, I called FIR (linphone_call_send_vfu_request) request but not helped.

Is there any config for OpenH264 video codec?

I want to see the video as soon as accept call.

Thank you.


Solution

    1. Change ms2plugins.cmake file.

    linphone-android/submodules/cmake-builder/builders/ms2plugins.cmake

    lcb_build_method("dummy")
    
    lcb_dependencies("ms2")
    
    if(ENABLE_AMRNB OR ENABLE_AMRWB)
    lcb_dependencies("msamr")
    endif()
    if(ENABLE_CODEC2)
        lcb_dependencies("mscodec2")
    endif()
    if(ENABLE_ISAC OR ENABLE_ILBC OR ENABLE_WEBRTC_AEC OR ENABLE_WEBRTC_AECM)
        lcb_dependencies("mswebrtc")
    endif()
    if(ENABLE_SILK)
        lcb_dependencies("mssilk")
    endif()
    if(ENABLE_OPENH264)
        lcb_dependencies("msopenh264")
    endif()
    if(ENABLE_WASAPI)
        lcb_dependencies("mswasapi")
    endif()
    
    lcb_dependencies("msx264")
    
    if(ENABLE_VIDEO AND (CMAKE_SYSTEM_NAME STREQUAL "WindowsPhone"))
        lcb_dependencies("mswp8vid")
    endif()
    if(ENABLE_VIDEO AND (CMAKE_SYSTEM_NAME STREQUAL "WindowsStore"))
        lcb_dependencies("mswinrtvid")
    endif()
    
    1. Change ffmpeg.cmake file.

    linphone-android/submodules/cmake-builder/builders/ffmpeg.cmake

    if(LINPHONE_BUILDER_PREBUILT_URL)
        set(FFMPEG_FILENAME "ffmpeg-0.10.2-${LINPHONE_BUILDER_ARCHITECTURE}.zip")
        file(DOWNLOAD "${LINPHONE_BUILDER_PREBUILT_URL}/${FFMPEG_FILENAME}" "${CMAKE_CURRENT_BINARY_DIR}/${FFMPEG_FILENAME}" STATUS FFMPEG_FILENAME_STATUS)
        list(GET FFMPEG_FILENAME_STATUS 0 FFMPEG_DOWNLOAD_STATUS)
        if(NOT FFMPEG_DOWNLOAD_STATUS)
            set(FFMPEG_PREBUILT 1)
        endif()
    endif()
    
    if(FFMPEG_PREBUILT)
        lcb_url("${CMAKE_CURRENT_BINARY_DIR}/${FFMPEG_FILENAME}")
        lcb_build_method("prebuilt")
    elseif(CMAKE_SYSTEM_NAME STREQUAL "WindowsStore")
        # Use prebuilt library in the source tree for Windows 10
        lcb_external_source_paths("build/ffmpeg")
    else()
        if(WIN32)
            set(EP_ffmpeg_PATCH_OPTIONS "--binary")
        endif()
    
        lcb_git_repository("git://git.linphone.org/ffmpeg.git")
        lcb_git_tag_latest("bc")
        lcb_git_tag("51aa587f7ddac63c831d73eb360e246765a2675f")
        lcb_external_source_paths("externals/ffmpeg")
        lcb_may_be_found_on_system(YES)
        lcb_ignore_warnings(YES)
        lcb_build_method("autotools")
    
        lcb_dependencies("x264")
    
        lcb_configure_options(
            "--disable-doc"
            "--disable-debug"
            "--enable-small"
            "--disable-zlib"
            "--disable-bzlib"
            "--disable-ffplay"
            "--disable-ffprobe"
            #"--disable-ffserver"
            "--disable-avdevice"
            "--disable-avfilter"
            "--disable-network"
            "--disable-avformat"
            "--disable-everything"
            "--enable-decoder=mjpeg"
            "--enable-encoder=mjpeg"
            # Disable video acceleration support for compatibility with older Mac OS X versions (vda, vaapi, vdpau).
            #"--disable-vda"
            "--disable-vaapi"
            "--disable-vdpau"
            "--ar=\$AR"
            "--cc=\$CC"
            "--nm=\$NM"
            "--extra-cflags=\$CFLAGS -w"
            "--extra-cxxflags=\$CXXFLAGS"
            "--extra-ldflags=\$LDFLAGS"
        )
        if(ENABLE_H263 OR IOS)
            lcb_configure_options(
                "--enable-decoder=h263"
                "--enable-encoder=h263"
            )
        endif()
        if(ENABLE_H263P OR IOS)
            lcb_configure_options("--enable-encoder=h263p")
        endif()
        if(ENABLE_MPEG4 OR IOS)
            lcb_configure_options(
                "--enable-decoder=mpeg4"
                "--enable-encoder=mpeg4"
            )
        endif()
        lcb_linking_type("--disable-static" "--enable-shared")
        set(FFMPEG_ARCH "${CMAKE_SYSTEM_PROCESSOR}")
    
        if(WIN32)
            set(FFMPEG_TARGET_OS "mingw32")
            set(FFMPEG_ARCH "i386")
            lcb_extra_cflags("-include windows.h")
            lcb_extra_ldflags("-static-libgcc")
            lcb_configure_options("--enable-runtime-cpudetect")
        else()
            if(APPLE)
                set(FFMPEG_TARGET_OS "darwin")
                if(IOS)
                    lcb_configure_options(
                        "--enable-decoder=h264"
                        "--disable-iconv"
                        "--disable-mmx"
                        "--enable-cross-compile"
                        "--cross-prefix=${SDK_BIN_PATH}/"
                        "--sysroot=${CMAKE_OSX_SYSROOT}"
                    )
                    lcb_make_options("RANLIB=\"\$RANLIB\"")
                    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
                        set(FFMPEG_ARCH "arm64")
                    else()
                        set(FFMPEG_ARCH "${CMAKE_SYSTEM_PROCESSOR}")
                    endif()
                    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7")
                        lcb_configure_options("--enable-neon" "--cpu=cortex-a8" "--disable-armv5te" "--enable-armv6" "--enable-armv6t2")
                    endif()
                else()
                    lcb_configure_options(
                        "--enable-runtime-cpudetect"
                        "--sysroot=${CMAKE_OSX_SYSROOT}"
                    )
                endif()
            elseif(ANDROID)
                get_filename_component(TOOLCHAIN_PATH "${CMAKE_LINKER}" DIRECTORY)
    
                    set(X264_INC "$CFLAGS -I${CMAKE_INSTALL_PREFIX}/include")
                        set(X264_LIB "$LDFLAGS -L${CMAKE_INSTALL_PREFIX}/lib")
    
    
    
                lcb_configure_options(
                            "--enable-gpl"
                            "--enable-libx264"
                            "--enable-encoder=libx264"
                    "--enable-decoder=h264"
                    "--disable-iconv"
                    "--disable-mmx"
                    "--enable-cross-compile"
                    #"--cross-prefix=${TOOLCHAIN_PATH}/"
                    "--sysroot=${CMAKE_SYSROOT}"
                )
                set(FFMPEG_TARGET_OS "linux")
                set(FFMPEG_ARCH "${CMAKE_SYSTEM_PROCESSOR}")
                lcb_make_options("RANLIB=\"\$RANLIB\"")
                if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")
                    lcb_configure_options("--enable-neon" "--cpu=cortex-a8" "--disable-armv5te" "--enable-armv6" "--enable-armv6t2")
    
                else()
                    lcb_configure_options("--disable-mmx" "--disable-sse2" "--disable-ssse3" "--disable-asm")
                endif()
    
                if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")
                    set(FFMPEG_CROSS_PREFIX "arm-linux-androideabi-")
                elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
                    set(FFMPEG_CROSS_PREFIX "aarch64-linux-android-")
                else()
                    set(FFMPEG_CROSS_PREFIX "i686-linux-android-")
                endif()
    
                if(CMAKE_C_COMPILER_TARGET) # When building with clang
                    lcb_configure_options("--extra-cflags=--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${_ANDROID_TOOL_C_COMPILER_EXTERNAL_TOOLCHAIN}")
                    lcb_configure_options("--extra-ldflags=--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${_ANDROID_TOOL_C_COMPILER_EXTERNAL_TOOLCHAIN}")
                endif()
            else()
                set(FFMPEG_TARGET_OS "linux")
                lcb_configure_options("--enable-runtime-cpudetect")
            endif()
            lcb_configure_options("--cc=$CC")
        endif()
    
        lcb_cross_compilation_options(
            "--cross-prefix=${TOOLCHAIN_PATH}/${FFMPEG_CROSS_PREFIX}"
            "--prefix=${CMAKE_INSTALL_PREFIX}"
            "--arch=${FFMPEG_ARCH}"
            "--target-os=${FFMPEG_TARGET_OS}"
                "--extra-cflags=${X264_INC}"
                "--extra-ldflags=${X264_LIB}"
        )
    
        if(ENABLE_X264)
            lcb_configure_options("--enable-decoder=h264")
        endif()
    endif()
    
    1. Change CMakeLists.txt file.

    linphone-android/submodules/externals/build/ffmpeg/CMakeLists.txt

    cmake_minimum_required(VERSION 3.0)
    project(ffmpegandroid LANGUAGES C)
    
    include(GNUInstallDirs)
    
    #JamesChan
    find_library(X246_LIBRARY x264)
    
    find_library(AVCODEC_LIBRARY avcodec)
    find_library(AVUTIL_LIBRARY avutil)
    find_library(SWRESAMPLE_LIBRARY swresample)
    find_library(SWSCALE_LIBRARY swscale)
    
    add_library(ffmpeg-linphone SHARED)
    
    #JamesChan
    target_link_libraries(ffmpeg-linphone LINK_PRIVATE "m" "-Wl,-whole-archive" ${AVCODEC_LIBRARY} ${AVUTIL_LIBRARY} ${SWRESAMPLE_LIBRARY} ${SWSCALE_LIBRARY} ${X246_LIBRARY} "-Wl,-no-whole-archive")
    
    #target_link_libraries(ffmpeg-linphone LINK_PRIVATE "m" "-Wl,-whole-archive" ${AVCODEC_LIBRARY} ${AVUTIL_LIBRARY} ${SWRESAMPLE_LIBRARY} ${SWSCALE_LIBRARY} "-Wl,-no-whole-archive")
    set_target_properties(ffmpeg-linphone PROPERTIES LINKER_LANGUAGE C)
    
    install(TARGETS ffmpeg-linphone
        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
    )
    
    1. Change x264.cmake file.

    linphone-android/submodules/cmake-builder/builders/x264.cmake

    lcb_git_repository("git://git.videolan.org/x264.git")
    lcb_git_tag("adc99d17d8c1fbc164fae8319b40d7c45f30314e")
    lcb_external_source_paths("externals/x264")
    lcb_ignore_warnings(YES)
    
    lcb_build_method("autotools")
    lcb_cross_compilation_options(
        "--prefix=${CMAKE_INSTALL_PREFIX}"
        "--host=${LINPHONE_BUILDER_HOST}"
    )
    lcb_configure_options(
        "--extra-cflags=$CFLAGS"
        "--extra-ldflags=$LDFLAGS"
    )
    if(IOS)
        lcb_configure_options("--sysroot=${CMAKE_OSX_SYSROOT}")
        string(REGEX MATCH "^(arm*|aarch64)" ARM_ARCH "${CMAKE_SYSTEM_PROCESSOR}")
        if(ARM_ARCH AND NOT ${XCODE_VERSION} VERSION_LESS 7)
            lcb_configure_options("--extra-asflags=-fembed-bitcode")
        endif()
    elseif(ANDROID)
        get_filename_component(TOOLCHAIN_PATH "${CMAKE_LINKER}" DIRECTORY)
        lcb_configure_options("--sysroot=${CMAKE_SYSROOT}")
        if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a")
                set(X264_HOST "arm-linux")
                set(X264_CROSS_PREFIX "arm-linux-androideabi-")
                set(X264_USE_ASM "--disable-asm")   
            elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
                set(X264_HOST "aarch64-linux")
                set(X264_CROSS_PREFIX "aarch64-linux-android-")
                set(X264_USE_ASM "")
            else()
                set(X264_HOST "i686-linux")
                set(X264_CROSS_PREFIX "i686-linux-android-")
                set(X264_USE_ASM "")
            endif()
    
        lcb_cross_compilation_options(
            "--prefix=${CMAKE_INSTALL_PREFIX}"
            "--host=${X264_HOST}"
            "--cross-prefix=${TOOLCHAIN_PATH}/${X264_CROSS_PREFIX}"
            "--enable-pic"
            ${X264_USE_ASM}
            #"--disable-cli"
            #"--disable-asm"
        )
    
        if(CMAKE_C_COMPILER_TARGET) # When building with clang
            lcb_configure_options("--extra-cflags=--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${_ANDROID_TOOL_C_COMPILER_EXTERNAL_TOOLCHAIN}")
            lcb_configure_options("--extra-ldflags=--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${_ANDROID_TOOL_C_COMPILER_EXTERNAL_TOOLCHAIN}")
        endif()
    
        lcb_use_c_compiler_for_assembler(YES)
    endif()
    
    lcb_linking_type("--enable-static")
    if(IOS)
        lcb_configure_env("CC=\"$CC -arch ${LINPHONE_BUILDER_OSX_ARCHITECTURES}\"")
    else()
        lcb_configure_env("CC=$CC")
    endif()
    lcb_install_target("install-lib-shared")
    

    5.Change ffmpeg-priv.h file if you used latest ffmpeg source code.

    linphone-iphone/submodules/mediastreamer2/src/utils/ffmpeg-priv.h

    Add following code.

    #define HAVE_FUN_avcodec_encode_video2 1
    #define HAVE_FUN_avcodec_get_context_defaults3 1
    #define HAVE_FUN_avcodec_open2 1
    
    1. Run prepare.py and make

      ./prepare.py -c && ./prepare.py -DENABLE_GPL_THIRD_PARTIES=NO -DENABLE_NON_FREE_CODECS=ON -DENABLE_VCARD=NO -DENABLE_DOC=NO -DENABLE_OPENH264=ON -DENABLE_X264=ON -DENABLE_EMBEDDED_OPENH264=ON -DENABLE_FFMPEG=ON -DENABLE_ILBC=NO -DENABLE_VPX=NO -DENABLE_MKV=NO -DENABLE_G729=NO -DENABLE_UNIT_TESTS=NO && make liblinphone-android-sdk

    IMPORTANT This build will not work on x64, only work on armv7-a.

    If there is exist any issue, please let me know. Thank you.