How would I send x264 encoded frames correctly over a network using UDP?

Question

I'm trying to send the encoded h264 frames I have over a network as I get them, currently I'm only streaming the nal units I get from each frame as it is encoded, is this the correct approach?

I wrote a receiver application on a different computer to get the nals and wrote them all to a file sequentially, when played with vlc I didn't get any video and instead just got a screeching noise. I'm not sure exactly where the problem would lie here. I have included the result of the FFmpeg -I command on the file created.

Encoder and sender code

    //Udp initialisation
    struct sockaddr_in broadcastAddr;
    int sock;
    int yes = 1;
    int addr_len;
    int count;
    fd_set readfd;
    char buffer[1024];
    int i;

    sock = socket(AF_INET, SOCK_DGRAM,0);

    if(sock < 0){
        std::cout << "Failed to initialise socket!" << std::endl;
    }

    int ret = setsockopt(sock, SOL_SOCKET, SO_BROADCAST, (char*)&yes, sizeof(yes));
    if(ret < 0){
        std::cout << "setsockopt error!" <<std::endl;
    }

    addr_len = sizeof(struct sockaddr_in); // the size of the address

    memset((void*)&broadcastAddr,0,addr_len); //0 out the address bits

    broadcastAddr.sin_family = AF_INET;
    broadcastAddr.sin_addr.s_addr = INADDR_BROADCAST;
    broadcastAddr.sin_port = PORT;



    //Set the encoder parameters
    x264_param_t param;
    x264_param_default_preset(&param,"veryfast","zerolatency");
    param.i_threads = 1;
    param.i_width = camera.getWidth();
    param.i_height = camera.getHeight();
    param.i_fps_num = 30;
    param.i_fps_den = 1;
// Intra refres:
    param.i_keyint_max = 30;
    param.b_intra_refresh = 1;
//Rate control:
    param.rc.i_rc_method = X264_RC_CRF;
    param.rc.f_rf_constant = 25;
    param.rc.f_rf_constant_max = 35;
//For streaming:
    param.b_repeat_headers = 1;
    param.b_annexb = 1;
    x264_param_apply_profile(&param, "baseline");

    x264_t *encoder = x264_encoder_open(&param); //H.264 encoder object
    x264_picture_t pic_in, pic_out;
    x264_picture_alloc(&pic_in, X264_CSP_I420,camera.getWidth(), camera.getHeight());

    //Network abstraction layer units for broadcast
    x264_nal_t *nals;
    int i_nals;

    while(true){

        //If there is valid data in the processing queue
        if(!encoderQueue.empty()){

            //File the x264 input data structure with the file data
            fillImage(encoderQueue.front(),camera.getWidth(),camera.getHeight(),&pic_in);

            //Encode and send
            int frame_size = x264_encoder_encode(encoder, &nals, &i_nals, &pic_in, &pic_out);
            if (frame_size >= 0) {
                //The frame is ready to be sent over UDP!
                for(int i = 0; i < i_nals; i++){
                    ret = sendto(sock, &nals[0].p_payload, frame_size,0,(struct sockaddr*)&broadcastAddr,addr_len);
                    if(ret > 0){
                        std::cout << "Streamed frame nal unit " << i  << std::endl;
                    } else{
                        std::cout << "Failed to stream nal unit " << i << std::endl;
                    }
                }
            }
            else{
                std::cout<<"Failed to encode h264 frame!" << std::endl;
            }
            //Finsihed with the current frame, pop it off the queue and remove any nals to do with it
            encoderQueue.pop();
            frame_size = 0;
            nals = nullptr;
            i_nals = 0;
        }


    }

Receiver application

#include <iostream>
#include <x264.h>
#include <Network/Network.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <queue>


#define BUFFER_LEN 10000
#define PORT_NO 3879



int main(int argc, const char * argv[]) {


    FILE *file; //File to write the h264 nals too

    //Declare the address memory space
    struct sockaddr_in sockAddr , bcAddr;
    socklen_t bcAddr_len = sizeof(&bcAddr); //Store the length of the broadcast address structure
    //0 out the assigned memory
    memset(&sockAddr, 0, sizeof(sockAddr));
    memset(&bcAddr, 0 ,sizeof(bcAddr));

    //Set the address parameters to look for incoming IpV4/UDP data
    sockAddr.sin_family = AF_INET;
    sockAddr.sin_port = htons(PORT_NO);
    sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);

    bcAddr.sin_family = AF_INET;
    bcAddr.sin_port = PORT_NO;
    inet_aton("255.255.255.255",&bcAddr.sin_addr);

    //Initialise a udp socket to read broadcast bytes
    int soc = socket(AF_INET, SOCK_DGRAM,0);

    //Check socket init
    if(soc < 0){
        std::cout << "Failed to initialise UDP socket!" << std::endl;
        return -1;
    }

    //Bind the address details to the socket, check for errors
    if(bind(soc, (struct sockaddr*)&sockAddr, sizeof(sockAddr)) < 0){
        std::cout << "Failed to bind address structure to socket!" << std::endl;
        return -2;
    }

    file = fopen("stream.h264","wb"); // Open the file for writing

    unsigned char buffer[BUFFER_LEN];

    while(true){


        memset(&buffer, 0, sizeof(unsigned char) * BUFFER_LEN);

        int recv_len = recvfrom(soc, buffer, BUFFER_LEN, 0, (struct sockaddr *)&bcAddr, &bcAddr_len);

        std::cout<< "Received " << recv_len << "bytes on broadcast address" << std::endl;

        fwrite(&buffer, sizeof(unsigned char), recv_len, file);
    }

    return 0;
}

FFMPEG -I output

FFMPEG -I output

Any help would be greatly appreciated.

Answer 1

There is no “correct” or “incorrect” approach, as long as the receiver and sender are in agreement with the protocol. In this example, VLC is looking for some sort of container or formating. Most commonly, a transport stream.

If you plan on reading the h264 stream and write it to disk, or pipe it to something expecting an annex b stream. What you are doing is correct. If you expect common players to read the stream directly, use a transport stream.