I'm learning how to use GStreamer on a Linux platform. I have gone through the basic tutorials and I think I understand what I've done there.
I'm now trying to modify the GTK+ integration tutorial (#5) so it uses a live video pipeline (v4l2src ! videoconvert ! ximagesink) instead of a playbin.
When I run it, my GTK+ application window opens and the stream goes into the PLAYING state but I don't see any video. If I comment off the call to gst_video_overlay_set_window_handle, then the ximagesink element opens another window, where I see the video working as expected.
So I don't think I have any problem with the pipeline itself, but there's something I haven't yet figured out about how to display the contents as an overlay in the GTK+ drawing area widget.
Here's a stripped-down version of the application as I currently have it:
#include <string.h>
#include <gtk/gtk.h>
#include <gst/gst.h>
#include <gst/video/videooverlay.h>
#include <gst/video/video.h>
#include <gdk/gdk.h>
#include <gdk/gdkx.h>
typedef struct CustomData
{
GstElement *pipeline;
GstElement *source;
GstElement *convert;
GstElement *sink;
GstState state; // Current stat of the pipeline
} CustomData;
static void realize_cb(GtkWidget *widget, CustomData *data)
{
GdkWindow *window;
guintptr window_handle;
window = gtk_widget_get_window(widget);
if (!gdk_window_ensure_native(window))
g_error ("Couldn't create native window needed for GstVideoOverlay!");
window_handle = GDK_WINDOW_XID(window);
// Comment off the next line and the app works, opening a new window
gst_video_overlay_set_window_handle(GST_VIDEO_OVERLAY(data->sink),
window_handle);
}
static void delete_event_cb(GtkWidget *widget,
GdkEvent *event,
CustomData *data)
{
gtk_main_quit();
}
static gboolean draw_cb(GtkWidget *widget, cairo_t *cr, CustomData *data)
{
if (data->state < GST_STATE_PAUSED)
{
GtkAllocation allocation;
gtk_widget_get_allocation(widget, &allocation);
cairo_set_source_rgb(cr, 0, 0, 0);
cairo_rectangle(cr, 0, 0, allocation.width, allocation.height);
cairo_fill(cr);
}
return FALSE;
}
static void create_ui(CustomData *data)
{
GtkWidget *main_window; // The uppermost window, containing all others
GtkWidget *video_window; // The drawing area where the video will be shown
GtkWidget *controls; // HBox to hold the buttons and slider
GtkWidget *main_box; // VBox to hold video window and controls
GtkWidget *play_button, *pause_button, *stop_button;
main_window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
g_signal_connect(G_OBJECT(main_window), "delete-event",
G_CALLBACK(delete_event_cb), data);
video_window = gtk_drawing_area_new();
g_signal_connect(G_OBJECT(video_window), "realize",
G_CALLBACK(realize_cb), data);
g_signal_connect(G_OBJECT(video_window), "draw",
G_CALLBACK(draw_cb), data);
main_box = gtk_box_new(GTK_ORIENTATION_VERTICAL, 0);
gtk_box_pack_start(GTK_BOX(main_box), video_window, TRUE, TRUE, 0);
gtk_container_add(GTK_CONTAINER(main_window), main_box);
gtk_window_set_default_size(GTK_WINDOW(main_window), 640, 480);
gtk_widget_show_all(main_window);
}
static void error_cb(GstBus *bus, GstMessage *msg, CustomData *data)
{
GError *err;
gchar *debug_info;
gst_message_parse_error(msg, &err, &debug_info);
g_printerr("Error received from element %s: %s\n",
GST_OBJECT_NAME(msg->src), err->message);
g_printerr("Debugging information; %s\n",
debug_info ? debug_info : "none");
g_clear_error(&err);
g_free(debug_info);
gtk_main_quit();
};
static void state_changed_cb(GstBus *bus, GstMessage *msg, CustomData *data)
{
GstState old_state, new_state, pending_state;
gst_message_parse_state_changed(msg, &old_state, &new_state,
&pending_state);
if (GST_MESSAGE_SRC(msg) == GST_OBJECT(data->pipeline))
{
data->state = new_state;
g_print("State set to %s:\n", gst_element_state_get_name(new_state));
}
}
int main(int argc, char *argv[])
{
CustomData data = {};
GstBus *bus;
gtk_init(&argc, &argv);
gst_init(&argc, &argv);
data.source = gst_element_factory_make("v4l2src", "source");
data.convert = gst_element_factory_make("videoconvert", "convert");
data.sink = gst_element_factory_make("ximagesink", "sink");
data.pipeline = gst_pipeline_new("pipeline");
gst_bin_add_many(GST_BIN(data.pipeline), data.source, data.convert,
data.sink, NULL);
gst_element_link_many(data.source, data.convert, data.sink, NULL);
g_object_set(data.source, "device", "/dev/video0", NULL);
create_ui(&data);
bus = gst_element_get_bus(data.pipeline);
gst_bus_add_signal_watch(bus);
g_signal_connect(G_OBJECT(bus), "message::error",
(GCallback)error_cb, &data);
g_signal_connect(G_OBJECT(bus), "message::state-changed",
(GCallback)state_changed_cb, &data);
gst_object_unref(bus);
gst_element_set_state(data.pipeline, GST_STATE_PLAYING);
gtk_main();
gst_element_set_state(data.pipeline, GST_STATE_NULL);
gst_object_unref(data.pipeline);
return 0;
}
Any help in this area will be most appreciated.
I got the answer from someone not on this group. Here's the answer. At the time the X window is realized, it may still be too early to assign a video sink to the overlay, and the specific GST element you need to bind it to may not be one that you created (e.g. it may have been created internally by the sink element you created).
To solve this problem, GST sinks that support overlay generate an explicit notification (via the bus sync mechanism) when it is the right time. Applications should register a bus sync handler and when the appropriate message (video overlay prepare window handle) is received, bind the X window to the source of that message.
See https://gstreamer.freedesktop.org/documentation/video/gstvideooverlay.html?gi-language=c
Here's updated code, which works (note changes to realize_cb and the new bus_sync_handler function):
#include <string.h>
#include <gtk/gtk.h>
#include <gst/gst.h>
#include <gst/video/videooverlay.h>
#include <gst/video/video.h>
#include <gdk/gdk.h>
#include <gdk/gdkx.h>
typedef struct CustomData
{
GstElement *pipeline;
GstElement *source;
GstElement *convert;
GstElement *sink;
GstState state; // Current state of the pipeline
guintptr video_window_handle;
} CustomData;
static void realize_cb(GtkWidget *widget, CustomData *data)
{
GdkWindow *window;
window = gtk_widget_get_window(widget);
if (!gdk_window_ensure_native(window))
{
g_error ("Couldn't create native window needed for GstVideoOverlay!");
}
data->video_window_handle = GDK_WINDOW_XID(window);
}
static GstBusSyncReply bus_sync_handler(GstBus *bus,
GstMessage *message,
CustomData *data)
{
// Ignore all but prepare-window-handle messages
if (!gst_is_video_overlay_prepare_window_handle_message(message))
{
return GST_BUS_PASS;
}
if (data->video_window_handle)
{
g_print("About to assign window to overlay\n");
gst_video_overlay_set_window_handle(
GST_VIDEO_OVERLAY(GST_MESSAGE_SRC(message)),
data->video_window_handle);
}
else
{
g_warning("Should have gotten a video window handle by now\n");
}
}
static void delete_event_cb(GtkWidget *widget,
GdkEvent *event,
CustomData *data)
{
gtk_main_quit();
}
static gboolean draw_cb(GtkWidget *widget, cairo_t *cr, CustomData *data)
{
if (data->state < GST_STATE_PAUSED)
{
GtkAllocation allocation;
gtk_widget_get_allocation(widget, &allocation);
cairo_set_source_rgb(cr, 0, 0, 0);
cairo_rectangle(cr, 0, 0, allocation.width, allocation.height);
cairo_fill(cr);
}
return FALSE;
}
static void create_ui(CustomData *data)
{
GtkWidget *main_window; // The uppermost window, containing all others
GtkWidget *video_window; // The drawing area where the video will be shown
GtkWidget *controls; // HBox to hold the buttons and slider
GtkWidget *main_box; // VBox to hold video window and controls
GtkWidget *play_button, *pause_button, *stop_button;
main_window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
g_signal_connect(G_OBJECT(main_window), "delete-event",
G_CALLBACK(delete_event_cb), data);
video_window = gtk_drawing_area_new();
g_signal_connect(G_OBJECT(video_window), "realize",
G_CALLBACK(realize_cb), data);
g_signal_connect(G_OBJECT(video_window), "draw",
G_CALLBACK(draw_cb), data);
main_box = gtk_box_new(GTK_ORIENTATION_VERTICAL, 0);
gtk_box_pack_start(GTK_BOX(main_box), video_window, TRUE, TRUE, 0);
gtk_container_add(GTK_CONTAINER(main_window), main_box);
gtk_window_set_default_size(GTK_WINDOW(main_window), 640, 480);
gtk_widget_show_all(main_window);
}
static void error_cb(GstBus *bus, GstMessage *msg, CustomData *data)
{
GError *err;
gchar *debug_info;
gst_message_parse_error(msg, &err, &debug_info);
g_printerr("Error received from element %s: %s\n",
GST_OBJECT_NAME(msg->src), err->message);
g_printerr("Debugging information; %s\n",
debug_info ? debug_info : "none");
g_clear_error(&err);
g_free(debug_info);
gtk_main_quit();
};
static void state_changed_cb(GstBus *bus, GstMessage *msg, CustomData *data)
{
GstState old_state, new_state, pending_state;
gst_message_parse_state_changed(msg, &old_state, &new_state,
&pending_state);
if (GST_MESSAGE_SRC(msg) == GST_OBJECT(data->pipeline))
{
data->state = new_state;
g_print("State set to %s:\n", gst_element_state_get_name(new_state));
}
}
int main(int argc, char *argv[])
{
CustomData data = {};
GstBus *bus;
gtk_init(&argc, &argv);
gst_init(&argc, &argv);
data.source = gst_element_factory_make("v4l2src", "source");
data.convert = gst_element_factory_make("videoconvert", "convert");
data.sink = gst_element_factory_make("ximagesink", "sink");
data.pipeline = gst_pipeline_new("pipeline");
gst_bin_add_many(GST_BIN(data.pipeline), data.source, data.convert,
data.sink, NULL);
gst_element_link_many(data.source, data.convert, data.sink, NULL);
g_object_set(data.source, "device", "/dev/video0", NULL);
create_ui(&data);
bus = gst_element_get_bus(data.pipeline);
gst_bus_set_sync_handler(bus, (GstBusSyncHandler)bus_sync_handler,
&data, NULL);
gst_bus_add_signal_watch(bus);
g_signal_connect(G_OBJECT(bus), "message::error",
(GCallback)error_cb, &data);
g_signal_connect(G_OBJECT(bus), "message::state-changed",
(GCallback)state_changed_cb, &data);
gst_object_unref(bus);
gst_element_set_state(data.pipeline, GST_STATE_PLAYING);
gtk_main();
gst_element_set_state(data.pipeline, GST_STATE_NULL);
gst_object_unref(data.pipeline);
return 0;
}