基于RV1126平台imx291分析 --- open及media graph分析

Linux v4l2架构学习总链接

基于RV1126平台imx291分析 --- 先研究media再分析应用调用文章中一开始就被media相关的吓退了。

一星期的时间去研究media相关知识，现在终于可以动手了

补充：

写完了这篇文章后，感觉open相关的其实可以忽略，这里也没有什么多余的open操作，唯一的v4l2_fh_open，vivi open中也分析了

这里更多的是media grpah，通过media graph可以遍历到每一个entity，这里主要学习如何遍历就可以了

rkcif_fh_open()

static int rkcif_fh_open(struct file *filp)
{
	struct video_device *vdev = video_devdata(filp);
	struct rkcif_vdev_node *vnode = vdev_to_node(vdev);
	struct rkcif_stream *stream = to_rkcif_stream(vnode);
	struct rkcif_device *cifdev = stream->cifdev;
	int ret;

	....

	/* Make sure active sensor is valid before .set_fmt() */
	ret = rkcif_update_sensor_info(stream);
	if (ret < 0) {
		v4l2_err(vdev,
			 "update sensor info failed %d\n",
			 ret);
		return ret;
	}
        ...
}

rkcif_fh_open() -> rkcif_update_sensor_info()

int rkcif_update_sensor_info(struct rkcif_stream *stream)
{
	struct rkcif_sensor_info *sensor, *terminal_sensor;
	struct v4l2_subdev *sensor_sd;
	int ret = 0;

        /*
         * 这里的sensor_sd是mipi csi的subdev
         */

	sensor_sd = get_remote_sensor(stream, NULL);
	if (!sensor_sd)
		return -ENODEV;
        ...
}

rkcif_fh_open() -> rkcif_update_sensor_info() -> get_remote_sensor()

struct media_pad *media_entity_remote_pad(const struct media_pad *pad)
{
	struct media_link *link;

        /*
         * 注意这里的links链表上link和backlink
         * 对于rkcif_mipi来说，没有source pad
         * 所以不会有link，只有backlink
         * 也就是说link->sink == pad
         * 返回的link->source 则是mipi csi 的source pad
         */

	list_for_each_entry(link, &pad->entity->links, list) {
		if (!(link->flags & MEDIA_LNK_FL_ENABLED))
			continue;

		if (link->source == pad)
			return link->sink;

		if (link->sink == pad)
			return link->source;
	}

	return NULL;

}

static struct v4l2_subdev *get_remote_sensor(struct rkcif_stream *stream, u16 *index)
{
	struct media_pad *local, *remote;
	struct media_entity *sensor_me;


        /*
         * rkcif_mipi 会产生4个video节点，每个video阶段对应的entity只有1个pad
         */

	local = &stream->vnode.vdev.entity.pads[0];
	if (!local)
		return NULL;
	remote = media_entity_remote_pad(local);
	if (!remote)
		return NULL;

	if (index)
		*index = remote->index;

        /*
         * 上面知道remote是mipi csi的source pad
         * 所以sensor_me就是 mipi csi 的 entity
         */

	sensor_me = remote->entity;

        /*
         * 知道了sensor_me就是 mipi csi 的 entity
         * 那么这里返回的就是mipi csi的subdev
         */

	return media_entity_to_v4l2_subdev(sensor_me);
}

回到rkcif_update_sensor_info继续分析

rkcif_fh_open() -> rkcif_update_sensor_info() -> sd_to_sensor()

int rkcif_update_sensor_info(struct rkcif_stream *stream)
{
	struct rkcif_sensor_info *sensor, *terminal_sensor;
	struct v4l2_subdev *sensor_sd;
	int ret = 0;

	...

	sensor = sd_to_sensor(stream->cifdev, sensor_sd);
	if (!sensor) {
		v4l2_err(&stream->cifdev->v4l2_dev, "get sensor for updating failed!\n");
		return -ENODEV;
	}
        ...
}

static struct rkcif_sensor_info *sd_to_sensor(struct rkcif_device *dev,
					      struct v4l2_subdev *sd)
{
	u32 i;

        /*
         * sensor[] 是怎么来的
         * 具体可以看前面的分析。notify->bound函数
         * 基于RV1126平台imx291分析 --- media部件连接 一
         * https://blog.csdn.net/ldl617/article/details/115720600
         */

	for (i = 0; i < dev->num_sensors; ++i)
		if (dev->sensors[i].sd == sd)
			return &dev->sensors[i];

	if (i == dev->num_sensors) {
		for (i = 0; i < dev->num_sensors; ++i) {
			if (dev->sensors[i].mbus.type == V4L2_MBUS_CCP2)
				return &dev->lvds_subdev.sensor_self;
		}
	}

	return NULL;
}

回到rkcif_update_sensor_info继续分析

rkcif_fh_open() -> rkcif_update_sensor_info()

int rkcif_update_sensor_info(struct rkcif_stream *stream)
{
        ...

        /*
         * sensor->sd是mipi csi的subdev
         */

	ret = v4l2_subdev_call(sensor->sd, video, g_mbus_config,
			       &sensor->mbus);
	if (ret && ret != -ENOIOCTLCMD)
		return ret;

        /*
         * 保存到active_sensor
         */

        stream->cifdev->active_sensor = sensor;
        ...
}

rkcif_fh_open() -> rkcif_update_sensor_info() -> g_mbus_config...

对于g_mbus_config肯定是一系列的调用。因为这里的3个subdev驱动都有这个代码，下面看看是如何依次调用到

首先是mipi csi的g_mbus_config

static struct v4l2_subdev *get_remote_sensor(struct v4l2_subdev *sd)
{
	struct media_pad *local, *remote;
	struct media_entity *sensor_me;

        /*
         * media_entity_remote_pad 和 media_entity_to_v4l2_subdev
         * 上面都分析过，这里就直接口述不再贴代码
         * 之前说过每个entity都可能会有link和backlink
         * 对于这里为什么区sink pad而不是source pad，这里说一下原因
         * -------------------方向--------------------> 
         * imx291 -> mipi csi phy -> mipi csi -> rkcif_mipi
         * 首先要知道我们的目的是向后寻找，也就是找mipi csi phy 
         * 而不是向前找rkcif_mipi
         * 所以这里是sink pad的时候，当前entity的link是不会满足的
         * 只有backlink才满足
         * 只有这样返回的remote的值才是mipi csi phy的source pad
         */

	local = &sd->entity.pads[RK_CSI2_PAD_SINK];
	remote = media_entity_remote_pad(local);
	if (!remote) {
		v4l2_warn(sd, "No link between dphy and sensor\n");
		return NULL;
	}

       
        /*
         * 这里的代码是bug吗？
         * 直接使用remote->entity不香吗？
         * sensor_me是mipi csi phy的entity
         */

	sensor_me = media_entity_remote_pad(local)->entity;

        /*
         * 这里返回的是mipi csi phy的subdev
         */
	return media_entity_to_v4l2_subdev(sensor_me);
}

static int csi2_g_mbus_config(struct v4l2_subdev *sd,
			      struct v4l2_mbus_config *mbus)
{
        /*
         * 得到了mipi csi phy的subdev
         */
        
	struct v4l2_subdev *sensor_sd = get_remote_sensor(sd);
	int ret;

        /*
         * 这里会调用mipi csi phy的g_mbus_config
         */

    	ret = v4l2_subdev_call(sensor_sd, video, g_mbus_config, mbus);
	if (ret)
		return ret;

	return 0;
}

接着是mipi csi phy的g_mbus_config

static int mipidphy_g_mbus_config(struct v4l2_subdev *sd,
				  struct v4l2_mbus_config *config)
{
	struct mipidphy_priv *priv = to_dphy_priv(sd);
        
        /*
         * 这里的分析同上
         * 对应的sensor_sd是imx291的subdev
         */

	struct v4l2_subdev *sensor_sd = get_remote_sensor(sd);
	struct mipidphy_sensor *sensor;

	if (!sensor_sd)
		return -ENODEV;
        
        /*
         * 这里要看 rockchip_mipidphy_notifier_bound
         * 基于RV1126平台imx291分析 --- media部件连接 三
         * https://blog.csdn.net/ldl617/article/details/115726576
         */

	sensor = sd_to_sensor(priv, sensor_sd);
    
        /*
         * 主意这里传入的是sd，而不是sensor
         * mipidphy_update_sensor_mbus中会再进行一次上面的操作
         */
	mipidphy_update_sensor_mbus(sd);
	*config = sensor->mbus;

	return 0;
}

static int mipidphy_update_sensor_mbus(struct v4l2_subdev *sd)
{
	struct mipidphy_priv *priv = to_dphy_priv(sd);
	struct v4l2_subdev *sensor_sd = get_remote_sensor(sd);
	struct mipidphy_sensor *sensor = sd_to_sensor(priv, sensor_sd);
	struct v4l2_mbus_config mbus;
	int ret;

        /*
         * 调用imx291到g_mbus_config
         */

	ret = v4l2_subdev_call(sensor_sd, video, g_mbus_config, &mbus);
	if (ret)
		return ret;
        ...
}

最后是调用imx291到g_mbus_config。

int rkcif_update_sensor_info(struct rkcif_stream *stream)
{
	...


        /*
         * 准备填充terminal_sensor
         */        


	terminal_sensor = &stream->cifdev->terminal_sensor;
	get_remote_terminal_sensor(stream, &terminal_sensor->sd);
	if (terminal_sensor->sd) {
		ret = v4l2_subdev_call(terminal_sensor->sd, video, g_mbus_config,
				       &terminal_sensor->mbus);
		if (ret && ret != -ENOIOCTLCMD)
			return ret;
	}
        ...
}

rkcif_fh_open() -> rkcif_update_sensor_info() -> get_remote_terminal_sensor()

static void stack_push(struct media_graph *graph,
		       struct media_entity *entity)
{
	if (graph->top == MEDIA_ENTITY_ENUM_MAX_DEPTH - 1) {
		WARN_ON(1);
		return;
	}
        

        /*
         * top值加1
         * link这里可能是link也可能是backlink
         */

	graph->top++;
	graph->stack[graph->top].link = entity->links.next;
	graph->stack[graph->top].entity = entity;
}

void media_graph_walk_start(struct media_graph *graph,
			    struct media_entity *entity)
{
	media_entity_enum_zero(&graph->ent_enum);
	media_entity_enum_set(&graph->ent_enum, entity);

        
        /*
         * 这里就是entity总数加1操作的原因
         * stack[0].entity = NULL
         */

	graph->top = 0;
	graph->stack[graph->top].entity = NULL;
	stack_push(graph, entity);
	dev_dbg(entity->graph_obj.mdev->dev,
		"begin graph walk at '%s'\n", entity->name);
}

static void get_remote_terminal_sensor(struct rkcif_stream *stream,
				       struct v4l2_subdev **sensor_sd)
{
	struct media_graph graph;
	struct media_entity *entity = &stream->vnode.vdev.entity;
	struct media_device *mdev = entity->graph_obj.mdev;
	int ret;

	/* Walk the graph to locate sensor nodes. */
	mutex_lock(&mdev->graph_mutex);

        
        /*
         * media_graph_walk_init
         * 之前分析过这个函数
         * 主要作用就是 graph.ent_enum.bmap申请一块内存空间
         * 这块空间的大小计算方式
         * 1. size = mdev->entity_internal_idx_max + 1 所有的entity数量加1
         *       加1的目的是要保存一个NULL用作top，后面分析会看到
         * 2. size = ALLGN(size, BITS_PER_LONG)
         * 3. size = (size/BITS_PER_LONG) * sizeof(long)
         */

	ret = media_graph_walk_init(&graph, mdev);
	if (ret) {
		mutex_unlock(&mdev->graph_mutex);
		*sensor_sd = NULL;
		return;
	}

	media_graph_walk_start(&graph, entity);
	...
}

上面执行后，会得到这样一个效果图

接着分析  rkcif_fh_open() -> rkcif_update_sensor_info() -> get_remote_terminal_sensor()

#define link_top(en)	((en)->stack[(en)->top].link)
#define stack_top(en)	((en)->stack[(en)->top].entity)

static struct media_entity *
media_entity_other(struct media_entity *entity, struct media_link *link)
{
        printk(KERN_CRIT "entity->name 0 is %s\n", entity->name);
        if (link->is_backlink)
                printk(KERN_CRIT "is backlink!\n");
        else
                printk(KERN_CRIT "is link!\n");
        if (link->source->entity == entity) {
                printk(KERN_CRIT "entity->name 1 is %s\n", link->sink->entity->name);
                return link->sink->entity;
        } else {
                printk(KERN_CRIT "entity->name 2 is %s\n", link->source->entity->name);
                return link->source->entity;
        }
}

static void media_graph_walk_iter(struct media_graph *graph)
{
	struct media_entity *entity = stack_top(graph);
	struct media_link *link;
	struct media_entity *next;

        
        /*
         * 转换得到media_link
         * link或者backlink都有可能
         */


	link = list_entry(link_top(graph), typeof(*link), list);

	/* The link is not enabled so we do not follow. */

        /*
         * 处理标志位没有ENABLED的link
         */

	if (!(link->flags & MEDIA_LNK_FL_ENABLED)) {

                /*
                 * 找到下一个link
                 */

		link_top(graph) = link_top(graph)->next;
		dev_dbg(entity->graph_obj.mdev->dev,
			"walk: skipping disabled link '%s':%u -> '%s':%u\n",
			link->source->entity->name, link->source->index,
			link->sink->entity->name, link->sink->index);
		return;
	}

	/* Get the entity in the other end of the link . */

        /*
         * 解析link或者backlink
         * 这里打算加些log方便分析，见上面
         */

	next = media_entity_other(entity, link);

	/* Has the entity already been visited? */

        /*
         * 判断entity是否被访问过
         */        

	if (media_entity_enum_test_and_set(&graph->ent_enum, next)) {
		link_top(graph) = link_top(graph)->next;
		dev_dbg(entity->graph_obj.mdev->dev,
			"walk: skipping entity '%s' (already seen)\n",
			next->name);
		return;
	}

	/* Push the new entity to stack and start over. */
        
        /*
         * 找到下一个link
         */

	link_top(graph) = link_top(graph)->next;

        /* 对应的link和entity再入栈 */


	stack_push(graph, next);
	dev_dbg(entity->graph_obj.mdev->dev, "walk: pushing '%s' on stack\n",
		next->name);
}

struct media_entity *media_graph_walk_next(struct media_graph *graph)
{
	struct media_entity *entity;

	if (stack_top(graph) == NULL)
		return NULL;

	/*
	 * Depth first search. Push entity to stack and continue from
	 * top of the stack until no more entities on the level can be
	 * found.
	 */
        
        /*
         * link_top(graph) 是links链表上的成员
         * stack_top(graph)->links 是links链表的链表头
         * 
         */

	while (link_top(graph) != &stack_top(graph)->links)
		media_graph_walk_iter(graph);

        /*
         * 最底层的就是我们需要的
         */

	entity = stack_pop(graph);
	dev_dbg(entity->graph_obj.mdev->dev,
		"walk: returning entity '%s'\n", entity->name);

	return entity;
}

static void get_remote_terminal_sensor(struct rkcif_stream *stream,
				       struct v4l2_subdev **sensor_sd)
{
	...
        
        /*
         * 这里千辛万苦的找到了imx291的subdev
         * 但是我们之前已经找到过imx291了，这里为什么还要再换一种方法呢？
         * 原因是之前的方法是rockchip的，没有通用性，
         * 这里是内核的，是通用的
         */


	while ((entity = media_graph_walk_next(&graph))) {
                
                /*
                 * 找到MEDIA_ENT_F_CAM_SENSOR标志的entity 退出
                 * 如果没有的话，这里会变成死循环吗？
                 */

		if (entity->function == MEDIA_ENT_F_CAM_SENSOR)
			break;
	}
        
        mutex_unlock(&mdev->graph_mutex);
	media_graph_walk_cleanup(&graph);

	if (entity)


                /*
                 * 通过entity找到subdev
                 */


		*sensor_sd = media_entity_to_v4l2_subdev(entity);
	else
		*sensor_sd = NULL;
}

media_entity_other加了log，应用执行open，底层log如下

[17666.981635] rkcif rkcif_mipi_lvds: begin graph walk at 'stream_cif_mipi_id0'
[17666.981641] entity->name 0 is stream_cif_mipi_id0
[17666.981660] is backlink!
[17666.981670] entity->name 2 is rockchip-mipi-csi2
[17666.981684] rkcif rkcif_mipi_lvds: walk: pushing 'rockchip-mipi-csi2' on stack
[17666.981686] entity->name 0 is rockchip-mipi-csi2
[17666.981696] is backlink!
[17666.981703] entity->name 2 is rockchip-mipi-dphy-rx
[17666.981715] rkcif rkcif_mipi_lvds: walk: pushing 'rockchip-mipi-dphy-rx' on stack
[17666.981718] entity->name 0 is rockchip-mipi-dphy-rx
[17666.981727] is link!
[17666.981735] entity->name 1 is rockchip-mipi-csi2
[17666.981747] rkcif rkcif_mipi_lvds: walk: skipping entity 'rockchip-mipi-csi2' (already seen)
[17666.981749] entity->name 0 is rockchip-mipi-dphy-rx
[17666.981760] is backlink!
[17666.981768] entity->name 2 is m01_f_imx291 1-001a
[17666.981780] rkcif rkcif_mipi_lvds: walk: pushing 'm01_f_imx291 1-001a' on stack
[17666.981783] entity->name 0 is m01_f_imx291 1-001a
[17666.981789] is link!
[17666.981796] entity->name 1 is rockchip-mipi-dphy-rx
[17666.981807] rkcif rkcif_mipi_lvds: walk: skipping entity 'rockchip-mipi-dphy-rx' (already seen)
[17666.981811] rkcif rkcif_mipi_lvds: walk: returning entity 'm01_f_imx291 1-001a'

对应的效果图如下

继续分析  rkcif_fh_open()

static int rkcif_fh_open(struct file *filp)
{
        ...

	ret = v4l2_fh_open(filp);
	if (!ret) {
		ret = v4l2_pipeline_pm_use(&vnode->vdev.entity, 1);
		if (ret < 0)
			vb2_fop_release(filp);
	}
        ...
}

对于v4l2_pipeline_pm_use下一遍文章分析