v4l2 与media 结构

v4l2_device

include/media/v4l2-device.h
struct v4l2_device {
    /* dev->driver_data points to this struct.
       Note: dev might be NULL if there is no parent device
       as is the case with e.g. ISA devices. */
    struct device *dev; //设备模型dev
#if defined(CONFIG_MEDIA_CONTROLLER)
    struct media_device *mdev;  //媒体设备，指向一个媒体控制器的指针
#endif
    /* used to keep track of the registered subdevs */
    struct list_head subdevs;   //保存所有子设备的链表
    /* lock this struct; can be used by the driver as well if this
       struct is embedded into a larger struct. */
    spinlock_t lock;    //锁
    /* unique device name, by default the driver name + bus ID */
    char name[V4L2_DEVICE_NAME_SIZE];   //独一无二的名字，默认是驱动名+总线ID
    /* notify callback called by some sub-devices. */
    void (*notify)(struct v4l2_subdev *sd,
            unsigned int notification, void *arg);  //通知链回调函数，被子设备调用
    /* The control handler. May be NULL. */
    struct v4l2_ctrl_handler *ctrl_handler; //控制句柄
    /* Device's priority state */
    struct v4l2_prio_state prio;    //设备的优先级状态，一般有后台，交互，记录三种优先级，依次变高
    /* BKL replacement mutex. Temporary solution only. */
    struct mutex ioctl_lock;    //ioctl操作的互斥量
    /* Keep track of the references to this struct. */
    struct kref ref;    //引用计数
    /* Release function that is called when the ref count goes to 0. */
    void (*release)(struct v4l2_device *v4l2_dev);  //设备释放函数
};

struct v4l2_device在v4l2子系统中处理核心位置，链表subdevs保存所有v4l2子设备，v4l2_device通常会被嵌入另一个特定的结构体，根据不同arm厂商做特定修改来来确定。在OK8MP中，v4l2_device被嵌入到mxc_md中

1.2v4l2_subdev 结构体

很多设备都需要与子设备进行交互，通常情况下它们都是 I2C 设备，但也有例外。v4l2_subdev 结构体被用于子设备管理。

每一个子设备驱动都必须有一个 v4l2_subdev 结构体，这个结构体可以作为独立的简单子设备存在，也可以嵌入到更大的结构体（自定义的子设备结构体）里面。通常会有一个由内核设置的低层次结构体（i2c_client，也就是上面说的 i2c 设备），它包含了一些设备数据。需要设置低级别结构的私有数据指针指向 v4l2_subdev 结构体，方便从低级别的结构体访问 v4l2_subdev 结构体，达到双向访问的目的，对于 i2c_client 来说，可以用 i2c_set_clientdata 函数来设置，其它的需使用与之相应的函数来完成设置。

 */
struct v4l2_subdev {
#if defined(CONFIG_MEDIA_CONTROLLER)
        struct media_entity entity;
#endif  
        struct list_head list;
        struct module *owner;
        bool owner_v4l2_dev;
        u32 flags;
        struct v4l2_device *v4l2_dev;
        const struct v4l2_subdev_ops *ops;
        const struct v4l2_subdev_internal_ops *internal_ops;
        struct v4l2_ctrl_handler *ctrl_handler;
        char name[V4L2_SUBDEV_NAME_SIZE];
        u32 grp_id;
        void *dev_priv;                 
        void *host_priv;
        struct video_device *devnode;
        struct device *dev; 
        struct fwnode_handle *fwnode; firm ware  node 
        struct list_head async_list;
        struct v4l2_async_subdev *asd;
        struct v4l2_async_notifier *notifier;
        struct v4l2_async_notifier *subdev_notifier;
        struct v4l2_subdev_platform_data *pdata;
};

1.3 struct mxc_md

drivers/staging/media/imx/imx8-media-dev.c
struct mxc_md {
        struct mxc_isi_info             mxc_isi[MXC_ISI_MAX_DEVS]; 
        struct mxc_mipi_csi2_info       mipi_csi2[MXC_MIPI_CSI2_MAX_DEVS];
        struct mxc_parallel_csi_info    pcsidev;
        struct mxc_sensor_info          sensor[MXC_MAX_SENSORS];  //摄像头
​
        int link_status;
        int num_sensors;
        int valid_num_sensors;
        unsigned int nr_isi;
        bool parallel_csi;
​
        struct media_device media_dev;  //媒体设备
        struct v4l2_device v4l2_dev;//v4l2 dev
        struct platform_device *pdev; //平台设备
​
        struct v4l2_async_notifier subdev_notifier;
        struct v4l2_async_subdev *async_subdevs[MXC_MAX_SENSORS];
};
​
​
struct mxc_isi_info {  
        struct v4l2_subdev *sd;
        struct media_entity *entity;
        struct device_node *node;
        u32 interface[MAX_PORTS];
​
        char vdev_name[MXC_NAME_LENS];
        char sd_name[MXC_NAME_LENS];
        int id;
};
​
struct mxc_mipi_csi2_info {
        struct v4l2_subdev *sd;
        struct media_entity *entity;
        struct device_node *node;
​
        char sd_name[MXC_NAME_LENS];
        int id;
        bool vchannel;
};
​
struct mxc_parallel_csi_info {
        struct v4l2_subdev *sd;
        struct media_entity *entity;
        struct device_node *node;
​
        char sd_name[MXC_NAME_LENS];
        int id;
};
​
struct mxc_sensor_info {
        int                             id;
        struct v4l2_subdev              *sd;
        struct v4l2_async_subdev asd;
        bool mipi_mode;
};
​

mxc_md会以平台驱动方式注册，sensor数组保存两个camera数据。

1.4 struct media_device

media_device：用于运行时数据流的管理，嵌入在 V4L2 device结构 内部，运行时的意思就是：一个 V4L2 device 下属可能有非常多同类型的子设备（两个或者多个 sensor、ISP 等），那么在设备运行的时候如何知道数据流需要用到哪一个类型的哪一个子设备呢。这个时候就需要 media_device 了，它为这一坨的子设备建立一条虚拟的连线，建立起来一个运行时的 pipeline（管道），并且可以在运行时动态改变、管理接入的设备。

struct media_device {
    /* dev->driver_data points to this struct. */
    struct device *dev; //media_device父设备
    struct media_devnode devnode; //设备结点
​
    char model[32]; //model name 模块名
    char serial[40]; //设备串口号，可选
    char bus_info[32]; //总线信息
    u32 hw_revision;
    u32 driver_version;
​
    u32 entity_id;
    struct list_head entities; //已注册媒体设备实体
​
    /* Protects the entities list */
    spinlock_t lock; //保护上来链表的锁
    /* Serializes graph operations. */
    struct mutex graph_mutex;
​
    int (*link_notify)(struct media_pad *source,
               struct media_pad *sink, u32 flags); //媒体设备连接上后的回调函数
};

1.4 struct media_entity

include/media/media-device.h
struct media_entity {
    struct list_head list; /* 将entity加到media_device的链表中 */
    struct media_device *parent;    /* 所属媒体设备，就是上面的struct media_device */
    u32 id;             /* Entity ID, unique in the parent media * device context， 在meida_device中，这个ID是唯一的 */
    const char *name;       /* Entity name 实体名 */
    u32 type;           /* Entity type (MEDIA_ENT_T_*) 类型 */
    u32 revision;           /* Entity revision, driver specific */
    unsigned long flags;        /* Entity flags (MEDIA_ENT_FL_*)  */
    u32 group_id;           /* Entity group ID 组ID，极少用 */
​
    u16 num_pads;           /* Number of sink and source pads entity所包含pad的个数 */
    u16 num_links;          /* Number of existing links, both* enabled and disabled link个数，包含使能与禁用的 */
    u16 num_backlinks;      /* Number of backlinks */
    u16 max_links;          /* Maximum number of links link的最大个数，可以调整 */
​
    struct media_pad *pads;     /* Pads array (num_pads elements)，pad数组，通常是两个元素，souce pad sink pad */
    struct media_link *links;   /* Links array (max_links elements) link数组，保存所有连接到source pad的link */
​
    const struct media_entity_operations *ops;  /* entity的ops */
​
    /* Reference counts must never be negative, but are signed integers on
     * purpose: a simple WARN_ON(<0) check can be used to detect reference
     * count bugs that would make them negative.
     */
    int stream_count;       /* Stream count for the entity. */
    int use_count;          /* Use count for the entity. */
​
    struct media_pipeline *pipe;    /* Pipeline this entity belongs to. */
​
    union {
        /* Node specifications */
        struct {
            u32 major;
            u32 minor;
        } v4l;
        struct {
            u32 major;
            u32 minor;
        } fb;
        struct {
            u32 card;
            u32 device;
            u32 subdevice;
        } alsa;
        int dvb;
​
        /* Sub-device specifications */
        /* Nothing needed yet */
    } info;
};
​

　struct media_entity中所包含的结构体

struct media_link { 
    struct media_pad *source;   /* Source pad */
    struct media_pad *sink;     /* Sink pad  */
    struct media_link *reverse; /* Link in the reverse direction */
    unsigned long flags;        /* Link flags (MEDIA_LNK_FL_*) */
};
​
struct media_pad {
    struct media_entity *entity;    /* 所属Entity */
    u16 index;          /* 当前pad在pads数组中的index */
    unsigned long flags;        /* Pad flags (MEDIA_PAD_FL_*) */
};
​
struct media_entity_operations {/* 只有一个回调函数，用来连接两个entity */
    int (*link_setup)(struct media_entity *entity,const struct media_pad *local, const struct media_pad *remote, u32 flags);
};
​

3 v4l2注册流程

在8MP中v4l2是以平台总线方式注册，平台驱动

drivers/staging/media/imx/imx8-media-dev.c
static const struct of_device_id mxc_md_of_match[] = {
        { .compatible = "fsl,mxc-md",},
        { /* sentinel */ },
};
MODULE_DEVICE_TABLE(of, mxc_md_of_match);
​
static struct platform_driver mxc_md_driver = {
        .driver = {
                .name = MXC_MD_DRIVER_NAME,
                .of_match_table = mxc_md_of_match,
        },
        .probe = mxc_md_probe,
        .remove = mxc_md_remove,
};

probe函数部分代码，有几个重要的流程分支

​
static int mxc_md_probe(struct platform_device *pdev)
{
        struct device *dev = &pdev->dev;
        struct device_node *nd = dev->of_node;
        struct v4l2_device *v4l2_dev;//v4l2核心结构体
        struct mxc_md *mxc_md;//顶层结构体
        int ret;
                mxc_md = devm_kzalloc(dev, sizeof(*mxc_md), GFP_KERNEL);//分配内存
        if (!mxc_md)
                return -ENOMEM;
​
        mxc_md->pdev = pdev;
        platform_set_drvdata(pdev, mxc_md);
              mxc_md->parallel_csi = of_property_read_bool(nd, "parallel_csi");
​
        /* register media device  */
        strlcpy(mxc_md->media_dev.model, "FSL Capture Media Device",
                sizeof(mxc_md->media_dev.model));
        mxc_md->media_dev.ops = &mxc_md_ops;
        mxc_md->media_dev.dev = dev;
​
        /* register v4l2 device */
        v4l2_dev = &mxc_md->v4l2_dev; //设置 v4l2_dev 结构体地址
        v4l2_dev->mdev = &mxc_md->media_dev; //指向mxc_mdmedia设备
        v4l2_dev->notify = mxc_sensor_notify;  
        
        
        media_device_init(&mxc_md->media_dev);
​
        ret = v4l2_device_register(dev, &mxc_md->v4l2_dev);//实际并未注册v4l2设备，仅完成v4l2初始化工作
        if (ret < 0) {
                v4l2_err(v4l2_dev, "Failed to register v4l2_device (%d)\n", ret);
                goto clean_md;
        }
                v4l2_async_notifier_init(&mxc_md->subdev_notifier);
        ret = mxc_md_register_platform_entities(mxc_md, dev->of_node); 注册实体
        if (ret < 0)
                goto clean_v4l2;
​
        ret = register_sensor_entities(mxc_md);注册实体
        if (ret < 0)
        {
                printk("##########%s:register_sensor_entities\n",__func__);
        }
//              goto clean_ents;
        if (mxc_md->num_sensors > 0) {
                mxc_md->subdev_notifier.ops = &sd_async_notifier_ops;
                mxc_md->valid_num_sensors = 0;
                mxc_md->link_status = 0;
​
                ret = v4l2_async_notifier_reeister(&mxc_md->v4l2_dev,
                                                   &mxc_md->subdev_notifier);
                if (ret < 0) {
                        dev_warn(&mxc_md->pdev->dev, "#######Sensor register failed\n");
                        return ret;
                }
​
                if (!mxc_md->link_status) {
                        if (mxc_md->valid_num_sensors > 0) {
                                ret = subdev_notifier_complete(&mxc_md->subdev_notifier);
                                建立连接
                               if (ret < 0)
                                {
                                        printk("##########%s:subdev_notifier_complete error\n",__func__);
                                        return 0;
                                        goto clean_ents;
                                }
                                mxc_md_clean_unlink_channels(mxc_md);
                        } else {
                                /* no sensors connected */
                                mxc_md_unregister_all(mxc_md);
                        }
                }
        }
        
  static int subdev_notifier_complete(struct v4l2_async_notifier *notifier)
{
        struct mxc_md *mxc_md = notifier_to_mxc_md(notifier);
        int ret;
​
        dev_dbg(&mxc_md->pdev->dev, "%s\n", __func__);
        mutex_lock(&mxc_md->media_dev.graph_mutex);
​
dump_stack();
        ret = mxc_md_create_links(mxc_md);
        if (ret < 0)
                goto unlock;
​
        mxc_md->link_status = 1;
​
        ret = v4l2_device_register_subdev_nodes(&mxc_md->v4l2_dev);
unlock:
        mutex_unlock(&mxc_md->media_dev.graph_mutex);
        if (ret < 0) {
                v4l2_err(&mxc_md->v4l2_dev, "%s error exit\n", __func__);
                return ret;
        }
​
        if (mxc_md->media_dev.devnode)
                return ret;
​
        return media_device_register(&mxc_md->media_dev);
}
      
        

media_device_register

int __must_check media_device_register(struct media_device *mdev)
{
    int ret;
​
    if (WARN_ON(mdev->dev == NULL || mdev->model[0] == 0))
        return -EINVAL;
​
    mdev->entity_id = 1;
    INIT_LIST_HEAD(&mdev->entities);
    spin_lock_init(&mdev->lock);
    mutex_init(&mdev->graph_mutex);
​
    /* Register the device node. 注册设备结点 */
    mdev->devnode.fops = &media_device_fops;
    mdev->devnode.parent = mdev->dev;
    mdev->devnode.release = media_device_release;
    ret = media_devnode_register(&mdev->devnode);   //media设备注册
    if (ret < 0)
        return ret;
​
    ret = device_create_file(&mdev->devnode.dev, &dev_attr_model);
    if (ret < 0) {
        media_devnode_unregister(&mdev->devnode);
        return ret;
    }
​
    return 0;
}
​

　媒体设备结点注册函数，这个函数主要完成三个工作

1. 找到可用次设备号

2. 注册字符设备

3. 注册媒体设备(挂载到media总线上)，生成/dev/media*结点

int __must_check media_devnode_register(struct media_devnode *mdev)
{
    int minor;
    int ret;
​
    /* Part 1: Find a free minor number 找一个可用的次设备号 */
    mutex_lock(&media_devnode_lock);
    minor = find_next_zero_bit(media_devnode_nums, MEDIA_NUM_DEVICES, 0);
    if (minor == MEDIA_NUM_DEVICES) {
        mutex_unlock(&media_devnode_lock);
        printk(KERN_ERR "could not get a free minor\n");
        return -ENFILE;
    }
​
    set_bit(minor, media_devnode_nums);
    mutex_unlock(&media_devnode_lock);
​
    mdev->minor = minor;
​
    /* Part 2: Initialize and register the character device 初始化、注册字符设备 */
    cdev_init(&mdev->cdev, &media_devnode_fops); /* open read write ioctrl /dev/media*结点时会调用media_devnode_fops的函数, 函数的主要实现或者说最终调用是mdev->devnode.fops = &media_device_fops */
    mdev->cdev.owner = mdev->fops->owner;
​
    ret = cdev_add(&mdev->cdev, MKDEV(MAJOR(media_dev_t), mdev->minor), 1);
    if (ret < 0) {
        printk(KERN_ERR "%s: cdev_add failed\n", __func__);
        goto error;
    }
​
    /* Part 3: Register the media device 注册一个媒体设备（挂载在mediabus总线上），在系统中生成了/dev/mediaX节点 */
    mdev->dev.bus = &media_bus_type;    //media总线
    mdev->dev.devt = MKDEV(MAJOR(media_dev_t), mdev->minor);    //设备号
    mdev->dev.release = media_devnode_release;  //设备结点释放函数
    if (mdev->parent)
        mdev->dev.parent = mdev->parent;
    dev_set_name(&mdev->dev, "media%d", mdev->minor);
    ret = device_register(&mdev->dev);  //注册设备
    if (ret < 0) {
        printk(KERN_ERR "%s: device_register failed\n", __func__);
        goto error;
    }
​
    /* Part 4: Activate this minor. The char device can now be used. */
    set_bit(MEDIA_FLAG_REGISTERED, &mdev->flags);
​
    return 0;
​
error:
    cdev_del(&mdev->cdev);
    clear_bit(mdev->minor, media_devnode_nums);
    return ret;
}
​

在媒体设备中有一个很重要的成员entity,两个设备间相连接需要用到pad。连接两个entity需要设备函数media_entity_create_link

drivers/media/mc/mc-entity.c
int media_create_pad_link(struct media_entity *source, u16 source_pad,
                         struct media_entity *sink, u16 sink_pad, u32 flags)
{
        struct media_link *link;
        struct media_link *backlink;
​
        BUG_ON(source == NULL || sink == NULL);
        BUG_ON(source_pad >= source->num_pads);
        BUG_ON(sink_pad >= sink->num_pads);
​
        link = media_add_link(&source->links);
        if (link == NULL)
                return -ENOMEM;
​
        link->source = &source->pads[source_pad];
        link->sink = &sink->pads[sink_pad];
        link->flags = flags & ~MEDIA_LNK_FL_INTERFACE_LINK;
​
        /* Initialize graph object embedded at the new link */
        media_gobj_create(source->graph_obj.mdev, MEDIA_GRAPH_LINK,
                        &link->graph_obj);
​
        /* Create the backlink. Backlinks are used to help graph traversal and
         * are not reported to userspace.
         */
        backlink = media_add_link(&sink->links);
        if (backlink == NULL) {
                __media_entity_remove_link(source, link);
                return -ENOMEM;
        }       
​
        backlink->source = &source->pads[source_pad];
        backlink->sink = &sink->pads[sink_pad];
        backlink->flags = flags;
        backlink->is_backlink = true;
​
        /* Initialize graph object embedded at the new link */
        media_gobj_create(sink->graph_obj.mdev, MEDIA_GRAPH_LINK,
                        &backlink->graph_obj);
​
        link->reverse = backlink;
        backlink->reverse = link;
​
        sink->num_backlinks++;
        sink->num_links++;
        source->num_links++;
​
        return 0;
}
EXPORT_SYMBOL_GPL(media_create_pad_link);
​

　　当media_entity_create_link函数被调用时，会分配一个link内存，link->souce指向source pad，link->sink指向sink pad，并会分配backlink实现与link同样的功能。 　　两个entity连接示意图如下　　

　每个entity都会有一个source pad、一个sink pad，link个数则不定，根据需要可能有多个，可能有一个，也可能没有。可以认为数据源的那个entity(输出数据)，这是是entity1为souce entity，entity2为sink entity。link和backlink本质上并没有什么区别，只是link属于entity1，而backlink属于entity2。并且实际使用link，而backlink当作备份

 

root@OK8MP:~# v4l2-ctl --list-devices
 ():
        /dev/v4l-subdev0
        /dev/v4l-subdev1
​
VIV (platform:viv0):
        /dev/video0
​
VIV (platform:viv1):
        /dev/video1

root@OK8MP:~# cat /sys/class/video4linux/*/name           
mxc-mipi-csi2.0
imx577_mipi 1-001a
mxc_isi.0.m2m
mxc_isi.0.capture
viv_v4l20
viv_v4l21

root@OK8MP:~# ls /dev/media
media0  media1  media2  

- entity 1: mxc_isi.0 (16 pads, 2 links)
            type V4L2 subdev subtype Unknown flags 0
        pad0: Sink
                <- "mxc-mipi-csi2.0":4 [ENABLED]
        pad1: Sink
        pad2: Sink
        pad3: Sink
        pad4: Sink
        pad5: Sink
        pad6: Sink
        pad7: Sink
        pad8: Sink
        pad9: Sink
        pad10: Sink
        pad11: Sink
        pad12: Source
                -> "mxc_isi.0.capture":0 [ENABLED]
        pad13: Source
        pad14: Source
        pad15: Sink
​
- entity 18: mxc_isi.0.capture (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video1
        pad0: Sink
                <- "mxc_isi.0":12 [ENABLED]
​
- entity 22: mxc-mipi-csi2.0 (8 pads, 2 links)
             type Node subtype V4L flags 0
             device node name /dev/v4l-subdev0
        pad0: Sink
                <- "imx577_mipi 1-001a":0 [ENABLED,IMMUTABLE]
        pad1: Sink
        pad2: Sink
        pad3: Sink
        pad4: Source
                -> "mxc_isi.0":0 [ENABLED]
        pad5: Source
        pad6: Source
        pad7: Source
​
- entity 31: imx577_mipi 1-001a (1 pad, 1 link)
             type V4L2 subdev subtype Sensor flags 0
             device node name /dev/v4l-subdev1
        pad0: Source
                [fmt:SBGGR12_1X12/0x0 field:none colorspace:jpeg]
                -> "mxc-mipi-csi2.0":0 [ENABLED,IMMUTABLE]
​

v4l2_mbus_framefmt

include/uapi/linux/v4l2-mediabus.h

/**     
 * struct v4l2_mbus_framefmt - frame format on the media bus
 * @width:      image width
 * @height:     image height
 * @code:       data format code (from enum v4l2_mbus_pixelcode)
 * @field:      used interlacing type (from enum v4l2_field)
 * @colorspace: colorspace of the data (from enum v4l2_colorspace)
 * @ycbcr_enc:  YCbCr encoding of the data (from enum v4l2_ycbcr_encoding)
 * @quantization: quantization of the data (from enum v4l2_quantization)
 * @xfer_func:  transfer function of the data (from enum v4l2_xfer_func)
 */     
struct v4l2_mbus_framefmt {
        __u32                   width;
        __u32                   height;
        __u32                   code;
        __u32                   field;
        __u32                   colorspace;
        __u16                   ycbcr_enc;
        __u16                   quantization;
        __u16                   xfer_func;
        __u16                   reserved[11];
};
V4L2_MBUS_FMT

    {0x0112, 0x0A},//csi_dt_fmt_h
    {0x0113, 0x0A},//csi_dt_fmt_l 输出格式
    
    {0x0136, 0x18}, {0x0137, 0x00}  inck