westerossink代码理解

Westeros是一个轻量级的Wayland合成器库。它使用Wayland协议，并设计为与使用Wayland合成器构建的应用程序兼容。

本文是阅读westeros里面的westerossink部分得来，westerossink是作为gstreamer的一个插件，使用硬件v4l2的能力进行解码和显示。

westerossink中包含两个主要的线程：

• wstEOSDetectionThread，负责detection EOS。
• wstVideoOutputThread，负责获取解码buffer，并且送给video server。

代码下载

git clone https://github.com/rdkcmf/westeros

westerossink初始化

gst_westeros_sink_init和gst_westeros_sink_class_init是在gobject创建的时候在g_type_create_instance中被调用的，因为gstreamer的element都是继承自gobject，在g_type_create_instance中先调用gst_westeros_sink_class_init，然后是gst_westeros_sink_init，在westerossink中，使用G_DEFINE_TYPE和GST_BOILERPLATE效果是一样的。

#ifdef USE_GST1
#define gst_westeros_sink_parent_class parent_class
G_DEFINE_TYPE (GstWesterosSink, gst_westeros_sink, GST_TYPE_BASE_SINK)
#else
GST_BOILERPLATE (GstWesterosSink, gst_westeros_sink, GstBaseSink, GST_TYPE_BASE_SINK)
#endif

gst_westeros_sink_class_init

gst_westeros_sink_class_init
	-> gst_westeros_sink_soc_class_init
		-> wstDiscoverVideoDecoder

gst_westeros_sink_class_init首先初始化下面几个函数指针，其他函数的实现都是空的：

gobject_class->finalize= gst_westeros_sink_finalize;
gobject_class->set_property= gst_westeros_sink_set_property;
gobject_class->get_property= gst_westeros_sink_get_property;
gstbasesink_class->render= GST_DEBUG_FUNCPTR (gst_westeros_sink_render);

然后install属性：

• window_set
• rectangle
• zorder
• opacity
• video_width
• video_height
• video_pts

对应代码片段：

enum
{
  PROP_0,
  PROP_WINDOW_SET,
  PROP_ZORDER,
  PROP_OPACITY,
  PROP_VIDEO_WIDTH,
  PROP_VIDEO_HEIGHT,
  PROP_ENABLE_TIMECODE,
  PROP_VIDEO_PTS,
  PROP_RES_PRIORITY,
  PROP_RES_USAGE
};

最后再调用gst_westeros_sink_soc_class_init，完成soc相关的初始化，install属性和创建signal，wstDiscoverVideoDecoder也是在gst_westeros_sink_soc_class_init中完成的。

gst_westeros_sink_init

gst_westeros_sink_init
	-> gst_westeros_sink_soc_init

gst_westeros_sink_init函数会先初始化GstWesterosSink对象，所有的成员变量在这里初始化，然后调用gst_westeros_sink_soc_init，初始化sink->soc的所有成员变量以及wayland相关的变量：

sink->windowX= DEFAULT_WINDOW_X;
sink->windowY= DEFAULT_WINDOW_Y;
sink->windowWidth= DEFAULT_WINDOW_WIDTH;
sink->windowHeight= DEFAULT_WINDOW_HEIGHT;


if ( gst_westeros_sink_soc_init( sink ) == TRUE )
{
    sink->registry= 0;
    sink->shell= 0;
    sink->compositor= 0;
    sink->surfaceId= 0;
    sink->vpc= 0;
    sink->vpcSurface= 0;
    sink->output= 0;
}

在这部分还有几个环境变量的读取，不同环境变量控制不同的功能：

WESTEROS_SINK_USE_FREERUN
WESTEROS_SINK_USE_NV21
WESTEROS_SINK_USE_GFX
WESTEROS_SINK_LOW_MEM_MODE
WESTEROS_SINK_DEBUG_FRAME

wstSetupInputBuffers

调用ioctl接口，完成对input buffer的初始化，这部分inputbuffer memory mode是V4L2_MEMORY_MMAP类型，同样在wstSetupOutputBuffers中也是通过v4l2的VIDIOC_REQBUFS请求buffer，buffer类型设置的是V4L2_MEMORY_DMABUF，这个请求的buffer没有带内存块，只是返回的一个v4l2_requestbuffers类型，不同的memory类型，会走不同的内存分配函数。

struct v4l2_control ctl;
struct v4l2_requestbuffers reqbuf;
struct v4l2_buffer *bufIn;
void *bufStart;

// 获取指定control的值
memset( &ctl, 0, sizeof(ctl));
ctl.id= V4L2_CID_MIN_BUFFERS_FOR_OUTPUT;
rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_G_CTRL, &ctl );

// 调用VIDIOC_REQBUFS向设备请求视频缓冲区，即初始化视频缓冲区
memset( &reqbuf, 0, sizeof(reqbuf) );
reqbuf.count= neededBuffers;
reqbuf.type= bufferType;
// reqbuf的memory是inputMemMode: V4L2_MEMORY_MMAP
reqbuf.memory= sink->soc.inputMemMode;
rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_REQBUFS, &reqbuf );

// 初始化inBuffers列表
sink->soc.inBuffers= (WstBufferInfo*)calloc( reqbuf.count, sizeof(WstBufferInfo) );

// 然后循环初始化inBuffers列表，map每一个buffer的内存映射
for( i= 0; i < reqbuf.count; ++i )
{
    bufIn= &sink->soc.inBuffers[i].buf;
    bufIn->type= bufferType;
    bufIn->index= i;
    bufIn->memory= sink->soc.inputMemMode;
    if ( sink->soc.isMultiPlane )
    {
        memset( sink->soc.inBuffers[i].planes, 0, sizeof(struct v4l2_plane)*WST_MAX_PLANES);
        bufIn->m.planes= sink->soc.inBuffers[i].planes;
        bufIn->length= 3;
    }
    // VIDIOC_QUERYBUF查询缓冲区的状态
    rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_QUERYBUF, bufIn );
    
    
    if ( sink->soc.inputMemMode == V4L2_MEMORY_MMAP )
    {
        // 映射内存
        bufStart= mmap( NULL,
                       memLength,
                       PROT_READ | PROT_WRITE,
                       MAP_SHARED,
                       sink->soc.v4l2Fd,
                       memOffset );
        sink->soc.inBuffers[i].start= bufStart;
        sink->soc.inBuffers[i].capacity= memLength;
    }
  

wstSetupInputBuffers是在gst_westeros_sink_soc_render中调用的，第一次调用后后面不在走这段代码：

wstSetupOutputBuffers

GEM（Graphics Execution Manager）即是linux DRM( Direct Rendering Manager)中用于完成memory管理的内核基础设施

在wstSetupOutputBuffers中有两种output模式，wstSetupOutputBuffersDmabuf中调用wstSVPCreateGemBuffer创建，会先打开DRM设备，通过drmIoctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &arg)申请到一个dumb的内存块，此时出现了handle，随后又通过drmPrimeHandleToFD将handle转换为fd。

if ( sink->soc.outputMemMode == V4L2_MEMORY_DMABUF )
{
    result= wstSetupOutputBuffersDmabuf( sink );
}
else if ( sink->soc.outputMemMode == V4L2_MEMORY_MMAP )
{
    result= wstSetupOutputBuffersMMap( sink );
}

wstDiscoverVideoDecoder

wstDiscoverVideoDecoder是打开video设备，查询device能力，然后创建westerossink的caps：

int rc, len, i, fd;
bool acceptsCompressed;
GstWesterosSink dummySink;
struct v4l2_exportbuffer eb;
struct dirent *dirent;

memset( &dummySink, 0, sizeof(dummySink) );
DIR *dir= opendir("/dev");
if ( dir )
{
    //打开video设备
    if ( (len > 5) && !strncmp( dirent->d_name, "video", 5 ) )
    {
        char name[256+10];
        struct v4l2_capability caps;
        uint32_t deviceCaps;

        strcpy( name, "/dev/" );
        strcat( name, dirent->d_name );
        GST_DEBUG("checking device: %s", name);
        fd= open( name, O_RDWR | O_CLOEXEC );
        
        //...
        
        // 查询video能力
        rc= IOCTL( fd, VIDIOC_QUERYCAP, &caps );

        //...

        wstGetInputFormats( &dummySink );
        acceptsCompressed= false;
        for( i= 0; i < dummySink.soc.numInputFormats; ++i)
        {
            if ( dummySink.soc.inputFormats[i].flags & V4L2_FMT_FLAG_COMPRESSED )
            {
                acceptsCompressed= true;
                //初始化插件的caps
                wstBuildSinkCaps( klass, &dummySink );
                break;
            }
        }
    }
}

wayland协议相关

gst_westeros_sink_change_state状态从READY到PAUSE的时候，连接display到wayland服务端，创建sink->queue，创建sink->surface

case GST_STATE_CHANGE_READY_TO_PAUSED:
{
    captureInit(sink);

    sink->eosEventSeen= FALSE;
    if ( gst_westeros_sink_backend_ready_to_paused(sink, &passToDefault) )
    {
        sink->rejectPrerollBuffers = !gst_base_sink_is_async_enabled(GST_BASE_SINK(sink));

        if ( !sink->display )
        {
            // connect to wayland server
            sink->display= wl_display_connect(NULL);
        }
        if ( sink->display )
        {
            sink->queue= wl_display_create_queue(sink->display);
            if ( sink->queue )
            {
                 // get registry
                sink->registry= wl_display_get_registry( sink->display );
                if ( sink->registry )
                {
                    wl_proxy_set_queue((struct wl_proxy*)sink->registry, sink->queue);
                    wl_registry_add_listener(sink->registry, &registryListener, sink);
                    wl_display_roundtrip_queue(sink->display,sink->queue);

                    sink->surface= wl_compositor_create_surface(sink->compositor);
                    printf("westeros-sink: ready-to-paused: surface=%p\n", (void*)sink->surface);
                    wl_proxy_set_queue((struct wl_proxy*)sink->surface, sink->queue);
                    wl_display_flush( sink->display );
                }
                else
                {
                    GST_ERROR("westeros-sink: ready-to-paused: unable to get display registry\n");
                }
            }
            else
            {
                GST_ERROR("westeros-sink: ready-to-paused: unable to create queue\n");
            }
        }
        else
        {
            GST_ERROR("westeros-sink: ready-to-paused: unable to create display\n");
        }

        if ( sink->vpc && sink->surface )
        {
            sink->vpcSurface= wl_vpc_get_vpc_surface( sink->vpc, sink->surface );
            if ( sink->vpcSurface )
            {
                wl_vpc_surface_add_listener( sink->vpcSurface, &vpcListener, sink );
                wl_proxy_set_queue((struct wl_proxy*)sink->vpcSurface, sink->queue);
                wl_vpc_surface_set_geometry( sink->vpcSurface, sink->windowX, sink->windowY,
                                            sink->windowWidth, sink->windowHeight );
                wl_display_flush( sink->display );
                printf("westeros-sink: ready-to-paused: done add vpcSurface listener\n");
            }
            else
            {
                GST_ERROR("westeros-sink: ready-to-paused: failed to create vpcSurface\n");
            }
        }
        else
        {
            GST_ERROR("westeros-sink: ready-to-paused: can't create vpc surface: vpc %p surface %p\n",
                      sink->vpc, sink->surface);
        }
    }
    else
    {
        result= GST_STATE_CHANGE_FAILURE;
    }
    break;
}

westerossink的printf输出可以看到这部分log，实际上如果不启用WESTEROS_SINK_USE_GFX，这个是没有用到的。

westeros-sink: registry: id 1 interface (wl_shm) version 1
westeros-sink: registry: id 2 interface (wl_compositor) version 3
westeros-sink: compositor 0xf012b448
westeros-sink: registry: id 3 interface (wl_shell) version 1
westeros-sink: registry: id 4 interface (xdg_shell) version 1
westeros-sink: registry: id 5 interface (wl_vpc) version 1
westeros-sink: registry: vpc 0xf0120e70
westeros-sink: registry: id 6 interface (wl_output) version 2
westeros-sink: registry: output 0xf0128318
westeros-sink: registry: id 7 interface (wl_seat) version 4
westeros-sink: registry: id 8 interface (mali_buffer_sharing) version 5
westeros-sink: registry: id 9 interface (wl_sb) version 3
westeros-sink-soc: registry: sb 0xf012aef0
westeros-sink-soc: registry: done add sb listener
westeros-sink: registry: id 10 interface (zwp_linux_dmabuf_v1) version 3
westeros-sink: registry: id 11 interface (wl_simple_shell) version 1
westeros-sink: shell 0xf012b2a0
westeros-sink: ready-to-paused: surface=0xf012ba40
westeros-sink: ready-to-paused: done add vpcSurface listener
westeros-sink: caps: (video/x-h264, width=(int)1920, height=(int)1080, parsed=(boolean)true, codec_id=(int)27, framerate=(fraction)24/1, alignment=(string)au, stream-format=(string)byte-stream, interlace-mode=(string)progressive, chroma-format=(string)4:2:0, bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8, profile=(string)constrained-baseline, level=(string)3)
westeros-sink: max frame (4096x2304)
wstSVPDecoderConfig: dw mode 16

从这段log来看，registry的是vpc，所以后面有done add vpcSurface listener的输出，在前面创建sink->surface之后又创建了sink->vpcSurface，对应代码：

if ( sink->vpc && sink->surface )
{
    sink->vpcSurface= wl_vpc_get_vpc_surface( sink->vpc, sink->surface );
    if ( sink->vpcSurface )
    {
        wl_vpc_surface_add_listener( sink->vpcSurface, &vpcListener, sink );
        wl_proxy_set_queue((struct wl_proxy*)sink->vpcSurface, sink->queue);
        wl_vpc_surface_set_geometry( sink->vpcSurface, sink->windowX, sink->windowY, sink->windowWidth, sink->windowHeight );
        wl_display_flush( sink->display );
        printf("westeros-sink: ready-to-paused: done add vpcSurface listener\n");
    }
    else
    {
        GST_ERROR("westeros-sink: ready-to-paused: failed to create vpcSurface\n");
    }
}

video client的创建

wstCreateVideoClientConnection创建video client，并连接到video server, 是在gst_westeros_sink_soc_ready_to_paused中调用wstCreateVideoClientConnection创建video client：

gboolean gst_westeros_sink_soc_ready_to_paused( GstWesterosSink *sink, gboolean *passToDefault )
{
   gboolean result= FALSE;

   WESTEROS_UNUSED(passToDefault);

   if ( sinkAcquireResources( sink ) )
   {
      wstStartEvents( sink );

      if ( !sink->soc.useCaptureOnly )
      {
         // DEFAULT_VIDEO_SERVER="video"
         sink->soc.conn= wstCreateVideoClientConnection( sink, DEFAULT_VIDEO_SERVER );
         if ( !sink->soc.conn )
         {
            GST_ERROR("unable to connect to video server (%s)", DEFAULT_VIDEO_SERVER );
            sink->soc.useCaptureOnly= TRUE;
            sink->soc.captureEnabled= TRUE;
            printf("westeros-sink: no video server - capture only\n");
            if ( sink->vpc )
            {
               wl_vpc_destroy( sink->vpc );
               sink->vpc= 0;
            }
         } else {
            wstSendLayerVideoClientConnection( sink->soc.conn );
         }
      }

      LOCK(sink);
      sink->startAfterCaps= TRUE;
      sink->soc.videoPlaying= TRUE;
      sink->soc.videoPaused= TRUE;
      UNLOCK(sink);

      result= TRUE;
   }
   else
   {
      GST_ERROR("gst_westeros_sink_ready_to_paused: sinkAcquireResources failed");
   }

   return result;
}

连接到video server

wstCreateVideoClientConnection创建video client链接到video server，video server的创建在westeros/drm/westeros-gl/westeros-gl.c文件中，wstInitServiceServer中可以看到，先获取XDG_RUNTIME_DIR，然后创建socket server。

static WstVideoClientConnection *wstCreateVideoClientConnection( GstWesterosSink *sink, const char *name )
{
   WstVideoClientConnection *conn= 0;
   int rc;
   bool error= true;
   const char *workingDir;
   int pathNameLen, addressSize;

   conn= (WstVideoClientConnection*)calloc( 1, sizeof(WstVideoClientConnection));
   if ( conn )
   {
      conn->socketFd= -1;
      conn->name= name;
      conn->sink= sink;

      // XDG_RUNTIME_DIR=/run/user/0
      workingDir= getenv("XDG_RUNTIME_DIR");
      if ( !workingDir )
      {
         GST_ERROR("wstCreateVideoClientConnection: XDG_RUNTIME_DIR is not set");
         goto exit;
      }

      pathNameLen= strlen(workingDir)+strlen("/")+strlen(conn->name)+1;
      if ( pathNameLen > (int)sizeof(conn->addr.sun_path) )
      {
         GST_ERROR("wstCreateVideoClientConnection: name for server unix domain socket is too long: %d versus max %d",
                pathNameLen, (int)sizeof(conn->addr.sun_path) );
         goto exit;
      }

      //sun_path经过strcat之后，video server就是：/run/user/0/video文件
      conn->addr.sun_family= AF_LOCAL;
      strcpy( conn->addr.sun_path, workingDir );
      strcat( conn->addr.sun_path, "/" );
      strcat( conn->addr.sun_path, conn->name );

      // 创建套接字, PF_LOCAL: Local to host (pipes and file-domain) 
      // SOCK_STREAM（流格式套接字/面向连接的套接字）
      conn->socketFd = socket( PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC, 0 );
      if ( conn->socketFd < 0 )
      {
         GST_ERROR("wstCreateVideoClientConnection: unable to open socket: errno %d", errno );
         goto exit;
      }

      addressSize= pathNameLen + offsetof(struct sockaddr_un, sun_path);

      // 通过connect()向socket server发起请求，server的IP地址和端口号保存在sockaddr_in结构体中。
      rc= connect(conn->socketFd, (struct sockaddr *)&conn->addr, addressSize );
      if ( rc < 0 )
      {
         GST_ERROR("wstCreateVideoClientConnection: connect failed for socket: errno %d", errno );
         goto exit;
      }

      error= false;
   }

exit:

   if ( error )
   {
      wstDestroyVideoClientConnection( conn );
      conn= 0;
   }

   return conn;
}

video server的创建

wstInitServiceServer初始化是在westeros的compositor里面逐级初始化的（代码在westeros/drm/westeros-gl/westeros-gl.c文件中），下面是调用栈：

wstCompositorCreateRenderer
	WstRendererCreate
		renderer_init
			wstRendererGLCreate
				WstGLInit
				wstRendererGLSetupEGL

	WstGLInit
		wstInitCtx
			wstInitVideoServer
				wstInitServiceServer

wstInitServiceServer中创建video server socket，run/user/0/video是套接字文件，在设备上可以找到，套接字文件属性位是s

sh-5.1# ls -l /run/user/0/video     
srwxrwxr-x. 1 root session 0 Mar 24 11:25 /run/user/0/video

wstInitServiceServer的代码：

static bool wstInitServiceServer( const char *name, WstServerCtx **newServer )
{
   bool result= false;
   const char *workingDir;
   int rc, pathNameLen, addressSize;
   WstServerCtx *server= 0;

   server= (WstServerCtx*)calloc( 1, sizeof(WstServerCtx) );
   if ( !server )
   {
      ERROR("No memory for server name (%s)", name);
      goto exit;
   }

   pthread_mutex_init( &server->mutex, 0 );
   server->socketFd= -1;
   server->lockFd= -1;
   server->name= name;

   ++server->refCnt;

   workingDir= getenv("XDG_RUNTIME_DIR");
   if ( !workingDir )
   {
      ERROR("wstInitServiceServer: XDG_RUNTIME_DIR is not set");
      goto exit;
   }

   pathNameLen= strlen(workingDir)+strlen("/")+strlen(server->name)+1;
   if ( pathNameLen > (int)sizeof(server->addr.sun_path) )
   {
      ERROR("wstInitServiceServer: name for server unix domain socket is too long: %d versus max %d",
             pathNameLen, (int)sizeof(server->addr.sun_path) );
      goto exit;
   }

   // server->addr.sun_path: run/user/0/video，这些文件都可以在设备上找到
   server->addr.sun_family= AF_LOCAL;
   strcpy( server->addr.sun_path, workingDir );
   strcat( server->addr.sun_path, "/" );
   strcat( server->addr.sun_path, server->name );

   strcpy( server->lock, server->addr.sun_path );
   strcat( server->lock, ".lock" );

   // server->lock: run/user/0/video.lock
   server->lockFd= open(server->lock,
                        O_CREAT|O_CLOEXEC,
                        S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP );
   if ( server->lockFd < 0 )
   {
      ERROR("wstInitServiceServer: failed to create lock file (%s) errno %d", server->lock, errno );
      goto exit;
   }

   // 使用文件锁加锁
   rc= flock(server->lockFd, LOCK_NB|LOCK_EX );
   if ( rc < 0 )
   {
      ERROR("wstInitServiceServer: failed to lock.  Is another server running with name %s ?", server->name );
      goto exit;
   }

   // 从文件系统中删除一个名称
   (void)unlink(server->addr.sun_path);

   // 创建套接字
   server->socketFd= socket( PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC, 0 );
   if ( server->socketFd < 0 )
   {
      ERROR("wstInitServiceServer: unable to open socket: errno %d", errno );
      goto exit;
   }

   addressSize= pathNameLen + offsetof(struct sockaddr_un, sun_path);

   // 在调用socket函数创建套接字的时候，内核还没有为套接字分配地址及其端口，采用bind函数进行绑定地址与端口
   rc= bind(server->socketFd, (struct sockaddr *)&server->addr, addressSize );
   if ( rc < 0 )
   {
      ERROR("wstInitServiceServer: Error: bind failed for socket: errno %d", errno );
      goto exit;
   }

   // listen函数的第一个参数即为要监听的socket描述字
   // 第二个参数为相应socket可以排队的最大连接个数。
   rc= listen(server->socketFd, 1);
   if ( rc < 0 )
   {
      ERROR("wstInitServiceServer: Error: listen failed for socket: errno %d", errno );
      goto exit;
   }

   *newServer= server;

   result= true;

exit:

   if ( !result )
   {
      server->addr.sun_path[0]= '\0';
      server->lock[0]= '\0';
   }

   return result;
}

westerossink渲染

在westerossink中，gst_westeros_sink_soc_render是继承自basesink的render函数，但是gst_westeros_sink_soc_render并没有真正的去做渲染，其首先是将把输入buffer放入v4l2，然后启动输入数据流。

gst_westeros_sink_render
	-> gst_westeros_sink_soc_render

gst_westeros_sink_soc_render代码：

// 复制数据到buffIndex对应的内存区域 
copylen= sink->soc.inBuffers[buffIndex].capacity;
if ( copylen > avail )
{
	copylen= avail;
}
memcpy( sink->soc.inBuffers[buffIndex].start, &inData[offset], copylen );
offset += copylen;
avail -= copylen;
// 把inBuffers数据放回缓存队列
rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_QBUF, &sink->soc.inBuffers[buffIndex].buf );

// streamingon
rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_STREAMON, &sink->soc.fmtIn.type );

if ( !gst_westeros_sink_soc_start_video( sink ) )
{
    GST_ERROR("gst_westeros_sink_soc_render: gst_westeros_sink_soc_start_video failed");
}

westerossink的输出线程

wstVideoOutputThread是decoder输出处理线程，通过ioctl接口，先queue输出buffer到v4l2，然后VIDIOC_STREAMON，启动输出数据流。

for( i= 0; i < sink->soc.numBuffersOut; ++i )
{
    if ( sink->soc.isMultiPlane )
    {
        for( j= 0; j < sink->soc.outBuffers[i].planeCount; ++j )
        {
            sink->soc.outBuffers[i].buf.m.planes[j].bytesused= sink->soc.outBuffers[i].buf.m.planes[j].length;
        }
    }
    // VIDIOC_QBUF: queue outBuffers，这个和queue inBuffers类似
    rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_QBUF, &sink->soc.outBuffers[i].buf );
    if ( rc < 0 )
    {
        GST_ERROR("wstVideoOutputThread: failed to queue output buffer: rc %d errno %d", rc, errno);
        UNLOCK(sink);
        postDecodeError( sink, "output enqueue error" );
        goto exit;
    }
    ++sink->soc.outQueuedCount;
    sink->soc.outBuffers[i].queued= true;
}

// streaming on
rc= IOCTL( sink->soc.v4l2Fd, VIDIOC_STREAMON, &sink->soc.fmtOut.type );


// export WESTEROS_SINK_USE_GFX=1后会走到wstProcessTextureWayland
if ( !sink->soc.useGfxSync || !sink->soc.conn )
{
    if ( sink->soc.enableTextureSignal )
    {
        gfxFrame= true;
        wstProcessTextureSignal( sink, buffIndex );
    }
    else if ( sink->soc.captureEnabled && sink->soc.sb )
    {
        // 处理texture
        if ( wstProcessTextureWayland( sink, buffIndex ) )
        {
            gfxFrame= true;
            buffIndex= -1;
        }
    }
}


// 线程主循环，从v4l2获取解码后的buffer index，每隔1ms查询一次
for( ; ; )
{

    // 前面是一些状态处理，capture_ready部分是从v4l2中dequeue buffer
    capture_ready:

        GST_DEBUG("wstVideoOutputThread: capture_ready");

        buffIndex= wstGetOutputBuffer( sink );

        if ( sink->soc.quitVideoOutputThread ) break;

        if ( buffIndex < 0 )
        {
            usleep( 1000 );
            continue;
        }
    
        // 发送buffer信息到video server，这个是正常渲染的主要逻辑
        if ( sink->soc.conn && !gfxFrame )
        {
            sink->soc.resubFd= sink->soc.prevFrame2Fd;
            sink->soc.prevFrame2Fd= sink->soc.prevFrame1Fd;
            sink->soc.prevFrame1Fd= sink->soc.nextFrameFd;
            sink->soc.nextFrameFd= sink->soc.outBuffers[buffIndex].fd;

            if ( wstSendFrameVideoClientConnection( sink->soc.conn, buffIndex ) )
            {
                buffIndex= -1;
            }
        }    	
}

westerossink中增加signal

定义一个新的signal，这里signal的class offset为NULL，如果不为NULL，可以参考appsrc的实现。

   g_signals[SIGNAL_GOTFRMAME]= g_signal_new( "decoded-frame",
                                              G_TYPE_FROM_CLASS(GST_ELEMENT_CLASS(klass)),
                                              (GSignalFlags) (G_SIGNAL_RUN_LAST),
                                              0,    /* class offset */
                                              NULL, /* accumulator */
                                              NULL, /* accu data */
                                              NULL,
                                              G_TYPE_NONE, /* return_type */
                                              1,
                                              G_TYPE_UINT64 /* frame time */
                                             );

在需要的地方发送signal：

g_signal_emit (G_OBJECT(sink), g_signals[SIGNAL_GOTFRMAME], 0, frameTime, NULL);

g_signal_emit的第三个参数是detail，如果没有写成NULL或者0，这时候会产生一个warning，阅读gsignal.c中代码，这时候这个信号不会发送成功。

 g_signal_emit (G_OBJECT(sink), g_signals[SIGNAL_GOTFRMAME], frameTime, NULL);

没有detail参数后输出warning log：

glib-2.62.6/gobject/gsignal.c:3196: signal id '52' does not support detail (21333000)

g_signal_emit的原型：

void
g_signal_emit (gpointer instance,
	       guint    signal_id,
	       GQuark   detail,
	       ...)

---

westeros-sink VPC

VPC是定义的一个wayland协议，VPC协议为compositor和client提供了一种机制，以在用于视频的解码路径的控制中进行协作。compositor可以请求client使用完全加速的硬件路径或视频作为图形路径。

wl_vpc_surface接口可以由wl_surface实现，用于显示视频，允许compositor控制视频的解码路径并调整其位置。

set_geometry_with_crop设置具有crop的frame的vpc surface的位置。x、y、width和heigth是屏幕坐标中的值，用于指定要与视频帧一起纹理化的矩形。

vpc(video path control)协议

<protocol name="wl_vpc">
  <copyright>
  </copyright>
  <interface name="wl_vpc" version="1">
    
    <description summary="Video path control">
      The video path control protocol provides a mechanism for the compositor and client to
      cooperate in the control of the decoding path used for video.  The compositor can
      request the client to use a fully accelerated hardware path or a video-as-graphics
      path.
    </description>
    
    <request name="get_vpc_surface">
      <arg name="id" type="new_id" interface="wl_vpc_surface"/>
      <arg name="surface" type="object" interface="wl_surface"/>
    </request>
    
  </interface>

  <interface name="wl_vpc_surface" version="2">

    <description summary="vpc surface">
    An interface that may be implemented by a wl_surface which is being used to
    present video which allows the compositor to control the decoding pathway
    being used for the video and to adjust its positioning.
    </description>

    <enum name="pathway">
      <entry name="unspecified" value="0"/>
      <entry name="hardware" value="1"/>
      <entry name="graphics" value="2"/>
    </enum>

    <enum name="crop">
      <entry name="denom" value="100000"/>
    </enum>

    <event name="video_path_change">
      <arg name="new_pathway" type="uint"/>
    </event>

    <event name="video_xform_change">
      <arg name="x_translation" type="int" summary="x-axis translation"/>
      <arg name="y_translation" type="int" summary="y-axis translation"/>
      <arg name="x_scale_num" type="uint"  summary="x scale factor numerator"/>
      <arg name="x_scale_denom" type="uint"  summary="x scale factor denominator"/>
      <arg name="y_scale_num" type="uint"  summary="y scale factor numerator"/>
      <arg name="y_scale_denom" type="uint"  summary="y scale factor denominator"/>
      <arg name="output_width" type="uint" summary="width of associated output"/>
      <arg name="output_height" type="uint" summary="height of associated output"/>
    </event>

    <request name="set_geometry">
      <arg name="x" type="int"/>region
      <arg name="y" type="int"/>
      <arg name="width" type="int"/>
      <arg name="height" type="int"/>
    </request>

    <request name="set_geometry_with_crop" since="2">
      <description summary="set geometry with crop">
      Set vpc surface geometry with the ability to crop
      the frame.  The x, y, width, and heigth are values in
      screen coordinates that specify the rectangle to
      be textured with the video frame.  The crop values, when
      divided by crop denom give the region of the frame to
      use as the texture.
      </description>
      <arg name="x" type="int"/>
      <arg name="y" type="int"/>
      <arg name="width" type="int"/>
      <arg name="height" type="int"/>
      <arg name="crop_x" type="int"/>
      <arg name="crop_y" type="int"/>
      <arg name="crop_width" type="int"/>
      <arg name="crop_height" type="int"/>
    </request>

  </interface>

</protocol>

westeros github