I am trying to create a real-time stream of my screen using ffmpeg and encoder h264_nvenc but there's 5-6 frames delay between my mouse move and mouse move on the translation(first 5-6 frames are black)
I need to somehow encode 1 image of my screen using h264_nvenc and receive it as valid frame(packet) or maybe decrease delay to 1-2 frames
Maybe there are some other methods to get the image right away
UPD: I realised that my latency is because of encoder and decoder frames delay so is there any way to completely remove this frame delays?(Added decoder init function and decode function)
I've done some tests and managed to receive image right away with decoder and encoder flush method but cant do this the second time
Encode function:
static void encode(AVCodecContext* enc_ctx, AVFrame* frame, AVPacket* pkt, AVPacket** new_packet)
{
int ret;
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0) {
fprintf(stderr, "Error sending a frame for encoding\n");
exit(1);
}
int size = 0;
while (ret >= 0) {
ret = avcodec_receive_packet(enc_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
//printf("count: %i size: %i\n", count, size);
return;
}
else if (ret < 0) {
fprintf(stderr, "Error during encoding\n");
exit(1);
}
size += pkt->size;
*new_packet = av_packet_clone(pkt);
av_packet_unref(pkt);
}
}
FFMpeg encoder initialization:
void InitializeFFmpegEncoder()
{
const AVCodec* codec;
int i, ret, x, y;
FILE* f;
// Encoder
codec = avcodec_find_encoder_by_name("h264_nvenc"); // hevc_nvenc h264_nvenc
if (!codec) {
fprintf(stderr, "Codec '%s' not found\n", "hevc");
exit(1);
}
/*codec = avcodec_find_encoder(AV_CODEC_ID_H265);
if (!codec) {
fprintf(stderr, "Codec '%s' not found\n", "hevc");
exit(1);
}*/
cout << "Encoder codec name: " << codec->name << endl;
cout << "1" << endl;
enc_c = avcodec_alloc_context3(codec);
if (!enc_c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
cout << "2" << endl;
pkt = av_packet_alloc();
if (!pkt)
exit(1);
cout << "3" << endl;
/* put sample parameters */
enc_c->bit_rate = 192000000;
/* resolution must be a multiple of two */
enc_c->width = 1920;
enc_c->height = 1080;
/* frames per second */
enc_c->time_base = AVRational(1, 1);
//enc_c->framerate = AVRational(60, 1);
/* emit one intra frame every ten frames
* check frame pict_type before passing frame
* to encoder, if frame->pict_type is AV_PICTURE_TYPE_I
* then gop_size is ignored and the output of encoder
* will always be I frame irrespective to gop_size
*/
enc_c->gop_size = 1;
enc_c->max_b_frames = 0;
enc_c->pix_fmt = AV_PIX_FMT_BGR0;
enc_c->keyint_min = 0;
if (codec->id == AV_CODEC_ID_H264)
{
//av_opt_set(enc_c->priv_data, "preset", "p1", 0);
//av_opt_set(enc_c->priv_data, "tune", "ull", 0);
//av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
//av_opt_set(enc_c->priv_data, "preset", "p1", 0);
//av_opt_set(enc_c->priv_data, "tune", "ull", 0);
//av_opt_set(enc_c->priv_data, "strict_gop", "1", 0);
//av_opt_set(enc_c->priv_data, "preset", "lossless", 0);
//av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
}
cout << "4" << endl;
/* open it */
ret = avcodec_open2(enc_c, codec, NULL);
if (ret < 0) {
printf("Could not open codec: %s\n", av_make_error_string((char[64])(0), 64, ret));
exit(1);
}
cout << "5" << endl;
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = AV_PIX_FMT_BGR0; // AV_PIX_FMT_YUV444P AV_PIX_FMT_ARGB
frame->width = enc_c->width;
frame->height = enc_c->height;
cout << "6" << endl;
ret = av_frame_get_buffer(frame, 0);
if (ret < 0) {
fprintf(stderr, "Could not allocate the video frame data\n");
exit(1);
}
cout << "7" << endl;
}
Decoder init function:
int InitializeFFmpegDecoder()
{
int ret;
const AVCodec* decoder = NULL;
enum AVHWDeviceType type;
int i;
decoder = avcodec_find_decoder_by_name("h264_cuvid"); // h264_cuvid hevc_cuvid
if (!decoder) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
cout << "Decoder name: " << decoder->name << endl;
if (!(decoder_ctx = avcodec_alloc_context3(decoder)))
return AVERROR(ENOMEM);
decoder_ctx->get_format = ffmpeg_GetFormat;
decoder_ctx->gop_size = 0;
decoder_ctx->max_b_frames = 0;
decoder_ctx->keyint_min = 0;
decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
//decoder_ctx->get_format = get_format;
if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) {
fprintf(stderr, "Failed to open codec for stream \n");
return -1;
}
if (ctx == NULL)
{
ctx = sws_getContext(1920, 1080,
AV_PIX_FMT_NV12, 1920, 1080, // AV_PIX_FMT_YUV420P AV_PIX_FMT_NV12
AV_PIX_FMT_BGR0, 0, 0, 0, 0);
}
}
Decode function:
static void decode(AVCodecContext* dec_ctx, AVFrame* frame, AVPacket* pkt, char* bgra_image)
{
int ret;
ret = avcodec_send_packet(dec_ctx, pkt);
if (ret < 0) {
fprintf(stderr, "Error sending a packet for decoding\n");
exit(1);
}
while (ret >= 0) {
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
char* outData[1] = { bgra_image }; // RGB24 have one plane
int outLinesize[1] = { 4 * 1920 }; // RGB stride
sws_scale(ctx, frame->data, frame->linesize, 0, 1080, (uint8_t* const*)outData, outLinesize);
//*new_frame = av_frame_clone(frame);
//av_frame_ref()
//break;
}
}
For h264_nvenc set option delay to 0
But for decoder h264_cuvid I don't know how to make 0 delay stream so I switched to DXVA2 and set max_b_frames to 0