cffmpeglibav
Does Buffer Source change the pixel format to `pix_fmt` or does it only read the input as if it was that format?
I am learning libav and related things.
I came across this example from the ffmpeg documentation:
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
If I specify a pix_fmt other than dec_ctx->pix_fmt (for example, rgb24), will this buffer source convert the pixel format to rgb24, or will it instead read the source as if it was in rgb24 format, thereby causing incorrect decoding?
And what happens if I set a different video_size?
Solution
Specifying pix_fmt to rgb24, converts the pixel format to rgb24.
Without much knowledge about filter graphs, we may execute the code, and check the output.
For displaying the output frame, I replaced the "Trivial ASCII grayscale display" with OpenCV method cv::imshow.
Setting up a project that uses both Libav and OpenCV requires some initial effort...
Here is the output frame with the original code (output is Grayscale):
Here is the output frame with AV_PIX_FMT_BGR24 pixel format (output is BGR):
Note: BGR is used (instead of RGB) because OpenCV convention for color ordering is BGR.
When BGR format is used there is a warning message:
Changing video frame properties on the fly is not supported by all filters.
It is probably not the best practice to set pix_fmt in such way...
Complete code:
//https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/filtering_video.c
/*
* Copyright (c) 2010 Nicolas George
* Copyright (c) 2011 Stefano Sabatini
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
/**
* @file
* API example for decoding and filtering
* @example filtering_video.c
*/
#define _XOPEN_SOURCE 600 /* for usleep */
#ifdef __GNUC__
#include <unistd.h>
#else
//Windows - use C++11 for usleep
#include <chrono>
#include <thread>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
}
#define DO_TEST_PIX_FMT_BGR24 //If defined, test pixel format BGR24
const char* filter_descr = "scale=78:24,transpose=cclock";
/* other way:
scale=78:24 [scl]; [scl] transpose=cclock // assumes "[in]" and "[out]" to be input output pads respectively
*/
static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
AVFilterContext* buffersink_ctx;
AVFilterContext* buffersrc_ctx;
AVFilterGraph* filter_graph;
static int video_stream_index = -1;
static int64_t last_pts = AV_NOPTS_VALUE;
static int open_input_file(const char* filename)
{
AVCodec* dec;
int ret;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char* filters_descr)
{
char args[512];
int ret = 0;
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
#ifdef DO_TEST_PIX_FMT_BGR24
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_BGR24, AV_PIX_FMT_NONE }; //Replace AV_PIX_FMT_GRAY8 with AV_PIX_FMT_BGR24
#else
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
#endif
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
#ifdef DO_TEST_PIX_FMT_BGR24
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, (int)AV_PIX_FMT_BGR24 /*dec_ctx->pix_fmt*/, //Replace dec_ctx->pix_fmt with AV_PIX_FMT_BGR24
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
#else
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
#endif
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
/*
* Set the endpoints for the filter graph. The filter_graph will
* be linked to the graph described by filters_descr.
*/
/*
* The buffer source output must be connected to the input pad of
* the first filter described by filters_descr; since the first
* filter input label is not specified, it is set to "in" by
* default.
*/
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
/*
* The buffer sink input must be connected to the output pad of
* the last filter described by filters_descr; since the last
* filter output label is not specified, it is set to "out" by
* default.
*/
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame* frame, AVRational time_base)
{
//int x, y;
//uint8_t* p0, * p;
int64_t delay;
const AVRational av_time_base_q = {1, AV_TIME_BASE};
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
delay = av_rescale_q(frame->pts - last_pts,
time_base, av_time_base_q);
if (delay > 0 && delay < 1000000)
#ifdef __GNUC__
usleep(delay);
#else
//Windows
//https://stackoverflow.com/questions/4184468/sleep-for-milliseconds
std::this_thread::sleep_for(std::chrono::microseconds(delay));
#endif
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
//p0 = frame->data[0];
//puts("\033c");
//for (y = 0; y < frame->height; y++) {
// p = p0;
// for (x = 0; x < frame->width; x++)
// putchar(" .-+#"[*(p++) / 52]);
// putchar('\n');
// p0 += frame->linesize[0];
//}
//fflush(stdout);
//Use OpenCV for display
#ifdef DO_TEST_PIX_FMT_BGR24
cv::Mat img = cv::Mat(frame->height, frame->width, CV_8UC3, frame->data[0], frame->linesize[0]); //3 Color channels (BGR)
#else
cv::Mat img = cv::Mat(frame->height, frame->width, CV_8UC1, frame->data[0], frame->linesize[0]); //1 Color channels (Grayscale)
#endif
cv::imshow("img", img);
cv::waitKey(1);
}
int main(int argc, char** argv)
{
int ret;
AVPacket* packet;
AVFrame* frame;
AVFrame* filt_frame;
if (argc != 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
packet = av_packet_alloc();
if (!frame || !filt_frame || !packet) {
fprintf(stderr, "Could not allocate frame or packet\n");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
/* read all packets */
while (1) {
if ((ret = av_read_frame(fmt_ctx, packet)) < 0)
break;
if (packet->stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0) {
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
}
av_frame_unref(frame);
}
}
av_packet_unref(packet);
}
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
av_packet_free(&packet);
if (ret < 0 && ret != AVERROR_EOF) {
//fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
char errbuf[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(errbuf, AV_ERROR_MAX_STRING_SIZE, ret);
fprintf(stderr, "Error occurred: %s\n", errbuf);
exit(1);
}
cv::destroyAllWindows();
exit(0);
}
- how to pass parameter as a pointer to an array in c
- TCP Socket - printing all chars of a string that are not \n in C not working
- GTK3 Set GtkButton size
- How to compile my own glibc C standard library from source and use it?
- Ninja Build System + gcc/clang doesn't output diagnostic colors
- C pointers understanding
- Strict aliasing violation in C
- Parser in C doesn't store values in a array
- What is the order of evaluation of printf(..) parameters?
- Compare floating point numbers as integers
- Trap representation
- Difference between sqrt(x) and pow(x,0.5)
- Is it possible to have a 32-bit pointer on x86-64 without undefined behavior?
- Can you explain this 'Hello world' program?
- how can i check file write permissions in C++ code?
- Why does my C program give an error when looping through an array?
- Printing char array with null character in the middle in C
- How do I read the source code for a C library in CLion
- pthread nice value setting for default scheduler in C
- Use a 3d array on a function (array that contains arrays that contain strings)
- frida-server being targeted by port scan?
- Are compound literals always lvalue?
- Why is the beginning of a C file stream called `SEEK_SET`?
- How to Take whitespace in Input in C
- "undefined reference to" errors when linking static C library with C++ code
- Audio timescale-pitch modification - Open source examples?
- What is the easiest way to create a secured network client in Linux (C) without any external libraries?
- Are there any non-twos-complement implementations of C?
- Read variable value from PE file
- C: Memory Allocation on Stack for Nested Structures