1 通过gst-launch-1.0大致实现播放与录像
1.1 命令
gst-launch-1.0 rtspsrc location=rtsp://admin:yangquan123@192.168.10.11:554/Streaming/Channels/101 ! \
rtph264depay ! h264parse ! nvv4l2decoder ! \
nvvideoconvert ! "video/x-raw(memory:NVMM),width=1280,height=720,format=I420" ! tee name=t ! \
queue ! nvv4l2h264enc ! h264parse ! qtmux ! filesink location=~/Desktop/file.mp4 t. ! \
queue ! nvegltransform ! nveglglessink window-width=1280 window-height=720 sync=false
播放视频sink一直存在，每隔5秒保存一次录像
15582303530
2 更换动态管道中间的某一元素
参考：Changing elements in a pipeline
这个参考来自于GStreamer官网，示例在管道Playing状态下，更换元素。所有代码在gst-dynamic-manipulation目录下。
is-live = true，窗口sink sync=false 属性下依然可以稳定操作
具体操作方法如下：
- 阻塞queue的src pad，通过GST_PAD_PROBE_TYPE_BLOCK_DOWNSTREAM
- 监听需要更换元素的src pad，通过GST_PAD_PROBE_TYPE_BLOCK |
GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM - 阻塞回调函数中，给需要更换元素的sink pad发送event事件
- 更换元素的src pad回调函数中进行
gst_element_set_state (element, GST_STATE_NULL)
gst_bin_remove (GST_BIN (pipeline), element);
gst_bin_add (GST_BIN (pipeline), add_element);
gst_element_link_many (conv_before, add_element, conv_after, NULL);
gst_element_set_state (add_element, GST_STATE_PLAYING);
注意：经过 gst_element_set_state (effect_a, GST_STATE_NULL);
gst_bin_remove (GST_BIN (pipeline), effect_a)操作后。
我对g_object对象查看引用总数，以及GST_IS_ELEMENT (effect_a)发现引用总数并不为0,而且effect_a是元素对象，但是并不能再次解引用（再次解引用会引起系统崩溃），可能gstreamer通过上面两步已经把所有内存释放（暂时没有找到相关资料支持此说法）
lieryang 877554 26.3 0.2 383756 14068 pts/0 Sl+ 13:04 0:13 ./01_changing_element
lieryang 877554 26.3 0.2 383756 14580 pts/0 Sl+ 13:04 3:57 ./01_changing_element
查看内存并没有明显的泄漏问题。具体代码如下：
/* filename: 01_changing_element.c */
#include <gst/gst.h>
GST_DEBUG_CATEGORY_STATIC (my_category);
#define GST_CAT_DEFAULT my_category
static gchar *opt_effects = NULL;
#define DEFAULT_EFFECTS "identity,exclusion,navigationtest," \
"agingtv,videoflip,vertigotv,gaussianblur,shagadelictv,edgetv"
static GstPad *queue_1_src_pad;
static GstElement *conv_before;
static GstElement *conv_after;
static GstElement *effect_a;
static GstElement *effect_b;
static GstElement *pipeline;
static gboolean effect_flag;
static GQueue effects = G_QUEUE_INIT;
static GstPadProbeReturn
event_probe_cb (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
{
GMainLoop *loop = user_data;
GstElement *cur_effect;
GstElement *next;
gint ret;
if (GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info)) != GST_EVENT_EOS)
return GST_PAD_PROBE_PASS;
gst_pad_remove_probe (pad, GST_PAD_PROBE_INFO_ID (info));
if (effect_flag == 0) {
effect_b = gst_element_factory_make ("gaussianblur", "effect_b");
if (gst_element_set_state (effect_a, GST_STATE_NULL) == GST_STATE_CHANGE_FAILURE)
g_printerr ("%s set to NULL state fail\n", GST_ELEMENT_NAME(effect_a));
/**
* 从bin中移除会有以下两个隐含操作：
* 1.断开元素链接
* 2.对元素进行了一次解引用
*/
//g_print ("GST_OBJECT_REFCOUNT_VALUE(effect_a) = %d\n", GST_OBJECT_REFCOUNT_VALUE(effect_a));
gst_bin_remove (GST_BIN (pipeline), effect_a);
//gst_object_unref (effect_a);
//g_print ("GST_OBJECT_REFCOUNT_VALUE(effect_a) = %d\n", GST_OBJECT_REFCOUNT_VALUE(effect_a));
/**
* 经过 gst_element_set_state (effect_a, GST_STATE_NULL);
* gst_bin_remove (GST_BIN (pipeline), effect_a); 操作
* 依据我对g_object对象的理解和查看引用总数、以及GST_IS_ELEMENT (effect_a)
* 发现引用总数并不为0,而且effect_a是元素对象
* @@@但是@@@并不能再次解引用（再次解引用会引起系统崩溃），可能gstreamer通过上面两步已经把所有内存释放
*/
g_object_unref (effect_a);
//g_print ("GST_IS_ELEMENT (effect_a) = %d\n", GST_IS_ELEMENT (effect_a));
/* 更换元素 */
gst_bin_add (GST_BIN (pipeline), effect_b);
gst_element_link_many (conv_before, effect_b, conv_after, NULL);
gst_element_set_state (effect_b, GST_STATE_PLAYING);
effect_flag = 1;
g_print ("effect_a -> effect_b\n");
}
else {
effect_a = gst_element_factory_make ("shagadelictv", "effect_a");
gst_element_set_state (effect_b, GST_STATE_NULL);
gst_bin_remove (GST_BIN (pipeline), effect_b);
gst_bin_add (GST_BIN (pipeline), effect_a);
gst_element_link_many (conv_before, effect_a, conv_after, NULL);
gst_element_set_state (effect_a, GST_STATE_PLAYING);
g_print ("effect_b -> effect_a\n");
effect_flag = 0;
}
return GST_PAD_PROBE_DROP;
}
static GstPadProbeReturn
pad_probe_cb (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
{
GstPad *srcpad, *sinkpad;
GST_DEBUG_OBJECT (pad, "pad is blocked now");
/* remove the probe first */
gst_pad_remove_probe (pad, GST_PAD_PROBE_INFO_ID (info));
if (effect_flag == 0 ) { /* 目前是effect_a */
srcpad = gst_element_get_static_pad (effect_a, "src");
sinkpad = gst_element_get_static_pad (effect_a, "sink");
}
else {
srcpad = gst_element_get_static_pad (effect_b, "src");
sinkpad = gst_element_get_static_pad (effect_b, "sink");
}
/* install new probe for EOS */
gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_BLOCK |
GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, event_probe_cb, user_data, NULL);
gst_object_unref (srcpad);
/* push EOS into the element, the probe will be fired when the
* EOS leaves the effect and it has thus drained all of its data */
gst_pad_send_event (sinkpad, gst_event_new_eos ());
gst_object_unref (sinkpad);
return GST_PAD_PROBE_OK;
}
static gboolean
timeout_cb (gpointer user_data)
{
gst_pad_add_probe (queue_1_src_pad, GST_PAD_PROBE_TYPE_BLOCK_DOWNSTREAM,
pad_probe_cb, user_data, NULL);
return TRUE;
}
static gboolean
bus_cb (GstBus * bus, GstMessage * msg, gpointer user_data)
{
GMainLoop *loop = user_data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_ERROR:{
GError *err = NULL;
gchar *dbg;
GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "6.dynamic-change-element");
gst_message_parse_error (msg, &err, &dbg);
gst_object_default_error (msg->src, err, dbg);
g_clear_error (&err);
g_free (dbg);
g_main_loop_quit (loop);
break;
}
default:
break;
}
return TRUE;
}
int
main (int argc, char **argv) {
GError *err = NULL;
GMainLoop *loop;
GstElement *videotestsrc, *capsfilter, *queue_1, *queue_2, *sink;
g_setenv("GST_DEBUG_DUMP_DOT_DIR", "./", TRUE);
gst_init (&argc, &argv);
/* gst初始化后初始 , 0日志字符表示无颜色输出， 1表示有颜色输出*/
GST_DEBUG_CATEGORY_INIT (my_category, "my category", 0, "This is my very own");
pipeline = gst_pipeline_new ("pipeline");
videotestsrc = gst_element_factory_make ("videotestsrc", NULL);
capsfilter = gst_element_factory_make ("capsfilter", "capsfilter");
queue_1 = gst_element_factory_make ("queue", "queue_1");
conv_before = gst_element_factory_make ("videoconvert", "conv_before");
effect_a = gst_element_factory_make ("shagadelictv", "effect_a");
conv_after = gst_element_factory_make ("videoconvert", "conv_after");
queue_2 = gst_element_factory_make ("queue", NULL);
sink = gst_element_factory_make ("ximagesink", NULL);
gst_util_set_object_arg (G_OBJECT (capsfilter), "caps",
"video/x-raw, width=320, height=240, "
"format={ I420, YV12, YUY2, UYVY, AYUV, Y41B, Y42B, "
"YVYU, Y444, v210, v216, NV12, NV21, UYVP, A420, YUV9, YVU9, IYU1 }");
queue_1_src_pad = gst_element_get_static_pad (queue_1, "src");
g_object_set (videotestsrc, "is-live", TRUE, NULL);
g_object_set (sink, "sync", FALSE, NULL);
gst_bin_add_many (GST_BIN (pipeline), videotestsrc, capsfilter, queue_1, conv_before, effect_a,
conv_after, queue_2, sink, NULL);
gst_element_link_many (videotestsrc, capsfilter, queue_1, conv_before, effect_a,
conv_after, queue_2, sink, NULL);
effect_flag = 0;
gst_element_set_state (pipeline, GST_STATE_PLAYING);
loop = g_main_loop_new (NULL, FALSE);
gst_bus_add_watch (GST_ELEMENT_BUS (pipeline), bus_cb, loop);
g_timeout_add_seconds (1, timeout_cb, loop);
g_main_loop_run (loop);
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
return 0;
}
以下是修改版本（依据文件02_dynamic_filter元素释放步骤）：
#include <gst/gst.h>
GST_DEBUG_CATEGORY_STATIC (my_category);
#define GST_CAT_DEFAULT my_category
static GstPad *queue_1_src_pad;
static GstElement *conv_before;
static GstElement *conv_after;
static GstElement *effect_a;
static GstElement *effect_b;
static GstElement *pipeline;
static gint probe = FALSE;
static gulong block_probe_id = 0;
static GstPadProbeReturn
event_probe_cb (GstPad * pad, GstPadProbeInfo * info, gpointer user_data) {
if (GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info)) != GST_EVENT_EOS)
return GST_PAD_PROBE_PASS;
gst_pad_remove_probe (pad, GST_PAD_PROBE_INFO_ID (info));
if (!g_atomic_int_compare_and_exchange (&probe, FALSE, TRUE)) {
g_print ("g_atomic\n");
return GST_PAD_PROBE_OK;
}
if (effect_a) {
gst_element_unlink_many (conv_after, effect_a, conv_before, NULL);
gst_bin_remove (GST_BIN (pipeline), effect_a);
gst_element_set_state (effect_a, GST_STATE_NULL);
g_print ("GST_OBJECT_REFCOUNT_VALUE(effect_a) = %d\n", GST_OBJECT_REFCOUNT_VALUE(effect_a));
gst_clear_object(&effect_a);
/**
* 经过 gst_element_set_state (effect_a, GST_STATE_NULL);
* gst_bin_remove (GST_BIN (pipeline), effect_a); 操作
* 依据我对g_object对象的理解和查看引用总数、以及GST_IS_ELEMENT (effect_a)
* 发现引用总数并不为0,而且effect_a是元素对象
* @@@但是@@@并不能再次解引用（再次解引用会引起系统崩溃），可能gstreamer通过上面两步已经把所有内存释放
*
* 通过其他示例，我知道如何解决这个问题：
* 元素创建完成之后进行 ref (造成这个问题的原因可能是因为浮点引用)
* 删除元素的方法：
* 1.断开链接
* 2.移除元素
* 3.设定NULL状态
* 4.解引用
*
* 引用数稳定在1,不能再次解引用，再次解应用内存错误 （为什么02_dynamic_filter引用计数为0？？？）
*
*/
/* 更换元素 */
effect_b = gst_element_factory_make ("gaussianblur", "effect_b");
gst_object_ref (effect_b);
gst_bin_add (GST_BIN (pipeline), effect_b);
gst_element_sync_state_with_parent (effect_b);
gst_element_link_many (conv_before, effect_b, conv_after, NULL);
g_print ("effect_a -> effect_b\n");
}
else if (effect_b){
gst_element_unlink_many (conv_after, effect_b, conv_before, NULL);
gst_bin_remove (GST_BIN (pipeline), effect_b);
gst_element_set_state (effect_b, GST_STATE_NULL);
g_print ("GST_OBJECT_REFCOUNT_VALUE(effect_b) = %d\n", GST_OBJECT_REFCOUNT_VALUE(effect_b));
gst_clear_object(&effect_b);
effect_a = gst_element_factory_make ("shagadelictv", "effect_a");
gst_object_ref (effect_a);
gst_bin_add (GST_BIN (pipeline), effect_a);
gst_element_sync_state_with_parent (effect_a);
gst_element_link_many (conv_before, effect_a, conv_after, NULL);
g_print ("effect_b -> effect_a\n");
}
/* 移除queue阻塞 */
gst_pad_remove_probe (queue_1_src_pad, block_probe_id);
block_probe_id = 0;
return GST_PAD_PROBE_DROP;
}
static GstPadProbeReturn
pad_probe_cb (GstPad * pad, GstPadProbeInfo * info, gpointer user_data)
{
GstPad *srcpad, *sinkpad;
if (effect_a) {
srcpad = gst_element_get_static_pad (effect_a, "src");
sinkpad = gst_element_get_static_pad (effect_a, "sink");
}
else if (effect_b){
srcpad = gst_element_get_static_pad (effect_b, "src");
sinkpad = gst_element_get_static_pad (effect_b, "sink");
}
/* install new probe for EOS */
gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_BLOCK |
GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, event_probe_cb, user_data, NULL);
gst_object_unref (srcpad);
/* push EOS into the element, the probe will be fired when the
* EOS leaves the effect and it has thus drained all of its data */
gst_pad_send_event (sinkpad, gst_event_new_eos ());
gst_object_unref (sinkpad);
return GST_PAD_PROBE_OK;
}
static gboolean
timeout_cb (gpointer user_data)
{
probe = FALSE;
block_probe_id = gst_pad_add_probe (queue_1_src_pad, GST_PAD_PROBE_TYPE_BLOCK_DOWNSTREAM,
pad_probe_cb, user_data, NULL);
return TRUE;
}
static gboolean
bus_cb (GstBus * bus, GstMessage * msg, gpointer user_data)
{
GMainLoop *loop = user_data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_ERROR:{
GError *err = NULL;
gchar *dbg;
GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "6.dynamic-change-element");
gst_message_parse_error (msg, &err, &dbg);
gst_object_default_error (msg->src, err, dbg);
g_clear_error (&err);
g_free (dbg);
g_main_loop_quit (loop);
break;
}
default:
break;
}
return TRUE;
}
int
main (int argc, char **argv) {
GError *err = NULL;
GMainLoop *loop;
GstElement *videotestsrc, *capsfilter, *queue_1, *queue_2, *sink;
g_setenv("GST_DEBUG_DUMP_DOT_DIR", "./", TRUE);
gst_init (&argc, &argv);
/* gst初始化后初始 , 0日志字符表示无颜色输出， 1表示有颜色输出*/
GST_DEBUG_CATEGORY_INIT (my_category, "my category", 0, "This is my very own");
pipeline = gst_pipeline_new ("pipeline");
videotestsrc = gst_element_factory_make ("videotestsrc", NULL);
capsfilter = gst_element_factory_make ("capsfilter", "capsfilter");
queue_1 = gst_element_factory_make ("queue", "queue_1");
conv_before = gst_element_factory_make ("videoconvert", "conv_before");
effect_a = gst_element_factory_make ("shagadelictv", "effect_a");
conv_after = gst_element_factory_make ("videoconvert", "conv_after");
queue_2 = gst_element_factory_make ("queue", NULL);
sink = gst_element_factory_make ("ximagesink", NULL);
gst_object_ref (effect_a);
gst_util_set_object_arg (G_OBJECT (capsfilter), "caps",
"video/x-raw, width=320, height=240, "
"format={ I420, YV12, YUY2, UYVY, AYUV, Y41B, Y42B, "
"YVYU, Y444, v210, v216, NV12, NV21, UYVP, A420, YUV9, YVU9, IYU1 }");
queue_1_src_pad = gst_element_get_static_pad (queue_1, "src");
g_object_set (videotestsrc, "is-live", TRUE, NULL);
g_object_set (sink, "sync", FALSE, NULL);
gst_bin_add_many (GST_BIN (pipeline), videotestsrc, capsfilter, queue_1, conv_before, effect_a,
conv_after, queue_2, sink, NULL);
gst_element_link_many (videotestsrc, capsfilter, queue_1, conv_before, effect_a,
conv_after, queue_2, sink, NULL);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
loop = g_main_loop_new (NULL, FALSE);
gst_bus_add_watch (GST_ELEMENT_BUS (pipeline), bus_cb, loop);
g_timeout_add_seconds (1, timeout_cb, loop);
g_main_loop_run (loop);
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
return 0;
}
3 动态增加/删除 file(rtsp)sink
参考： How to add/delete rtsp sink during runtime?
上面链接NVIDIA论坛zongxp提问到：
我已经实现了在运行时为deepstream-app添加/删除source。现在我想要对sink实现相同的功能，而不是source。
在github的runtime_source_add_add_delete中，sink只是一个平铺显示。我想在不同的rtsp端口显示它。
你能给我一些关于这个功能的建议吗?
NVIDIA相关人员回复：并没有实现这个功能（我猜测动态操作sink可能并不稳定）。
prominence_ai回复：你可以这样做，具体可以参考GStreamer Dynamic Pipelines和Github代码链接
srcpad = gst_element_get_static_pad (tee, pad_name);
gst_pad_send_event (srcpad, gst_event_new_eos());
gst_element_release_request_pad (tee, srcpad);
gst_object_unref (srcpad);
# remove sink bin and unref
上述博客中主要使用的是GST_PAD_PROBE_TYPE_IDLE标志去添加的监听函数。（无论IDLE还是BLOCK，回调函数执行过程中，都是暂会阻塞数据传输，函数执行完之后是否阻塞，取决于函数返回值和FLAG）。
我认为上面博客中删除元素的方法，最后元素的引用计数值比较稳定（较好的释放内存）。
元素创建完成之后进行 ref (IDLE监听函数下，被释放元素引用计数=0,BLOCK/EVENT监听函数下，被释放元素引用计数=1，虽然是1,但是不能再次解引用了)
删除元素的方法：
- 断开链接
- 移除元素
- 设定NULL状态
- 解引用
待补充
智能录像（检测到目标进行录像）
方案1 NVDIA Smart Video
https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Smart_video.html
讨论了关于动态add remove filesink
https://forums.developer.nvidia.com/t/how-to-dynamically-add-remove-filesink/108821/41
Dynamically updating filesink location at run-time, on the fly
https://gstreamer-devel.narkive.com/2nMJgA88/dynamically-updating-filesink-location-at-run-time-on-the-fly
GSTREAMER access video before an event ( 参考意义不大)
https://stackoverflow.com/questions/52098093/gstreamer-access-video-before-an-event
5 Debug 问题
5.1
目前使用gstreamer version = 1.16.3
:00:04.824886967 45630 0xffff58037b60 WARN bufferpool gstbufferpool.c:1235:default_reset_buffer:<nvv4l2_decoder:pool:sink> Buffer 0xffff300a4120 without the memory tag has maxsize (0) that is smaller than the configured buffer pool size (4194304). The buffer will be not be reused. This is most likely a bug in this GstBufferPool subclass
0:00:04.833062287 45630 0xaaaad98cb0c0 WARN bufferpool gstbufferpool.c:1235:default_reset_buffer:<nvv4l2_h264enc:pool:sink> Buffer 0xffff300abc60 without the memory tag has maxsize (192) that is smaller than the configured buffer pool size (1382400). The buffer will be not be reused. This is most likely a bug in this GstBufferPool subclass
我在NVIDIA官网找到了相关问题的提问，可是通过降低版本并不能解决。gstbufferpool.c文件并不在gst-plugins-good里面，而是gst文件夹里面，是一个核心库里面，并不是good plugins里面。
我下载编译了1.19.3版本的gstreamer（建议使用1.16.3版本），注释掉了警告部分。通过查看源码中的注释，提到
检查我们是否可以调整到最小配置的池大小。如果不能,
然后这将在gst_buffer_resize()内部失败。
如果尺寸不匹配，default_release_buffer()将从缓冲区池中删除缓冲区
据此，应该不会造成内存泄漏，故注释掉了此部分代码。
2675
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
