Linux存储IO栈（3）-- 设备驱动模型

概述

Linux的设备驱动模型能够带来以下的优点：
* 使用统一机制来表达设备与驱动之间的关系，规范设备驱动的编写，核心代码复用。
* 将系统中的设备以树结构组织，并且通过sysfs将其呈现在用户空间——包括所有的总线和内部连接。
* 支持设备的热拔插机制。
* 支持通用的电源管理机制，通过由叶子节点到根节点的方向遍历设备树，确保子设备在父设备之前断电。

内核基于内核对象和sysfs，通过抽象以下五种概念，实现了设备驱动模型的框架，使得编写子系统成为“八股文”。
1. bus_type: 总线类型，每个子系统有且只有一个总线类型，由bus_type和subsys_private两个结构共同描述。
2. device: 设备，描述挂在总线类型中的设备，由device和device_private两个结构共同描述。
3. driver: 驱动， 描述挂在总线类型中的驱动模块，由device_driver和driver_private两个结构共同描述。
4. class: 类，每个总线类型有且只有一个类，由class和subsys_private两个结构共同描述。
5. class_interface: 接口，每个类有多个接口，由class_interface结构描述。

在Linux内核中，子系统是由bus_type, device, driver, class和class_interface之间的关系所描述，而设备驱动模型正是这些关系的核心实现，使得在编写子系统程序时，只要遵循设备模型的套路，便不需要关注于这些复杂的关系，只需实现自身的业务逻辑。

每个子系统都有一个总线类型，总线类型拥有一个设备链表和一个驱动链表，用于连接由该总线类型已发现的设备和已加载的驱动，设备发现和驱动加载的顺序是任意的。每个设备最多绑定到一个驱动，被绑定了驱动的设备可以正常工作。除此之外，每个设备可以唯一属于某个类，类中包含多个接口，接口的方法作用于设备，不管是先添加接口，还是先发现设备。

总线类型

总线类型的数据结构

struct bus_type {
    const char      *name;         //子系统名称
    const char      *dev_name;     //供子系统生成设备名称使用
    struct device       *dev_root;
    struct device_attribute *dev_attrs; /* use dev_groups instead */
    const struct attribute_group **bus_groups;  //总线类型使用的属性组
    const struct attribute_group **dev_groups;  //设备使用的属性组
    const struct attribute_group **drv_groups;  //驱动使用的属性组

    int (*match)(struct device *dev, struct device_driver *drv);    //检测设备与驱动是否可以绑定
    int (*uevent)(struct device *dev, struct kobj_uevent_env *env); //发送事件前，设置bus特有的环境变量
    int (*probe)(struct device *dev);     //当设备可以绑定到驱动时，对设备进行初始化和执行绑定
    int (*remove)(struct device *dev);    //当设备从驱动中解绑时，回调
    void (*shutdown)(struct device *dev); //当设备断电时，回调

    int (*online)(struct device *dev);    //当设备上电时，回调
    int (*offline)(struct device *dev);   //当设备下电时，回调

    int (*suspend)(struct device *dev, pm_message_t state); //当设备进入节能状态时，回调
    int (*resume)(struct device *dev);                      //当设备恢复正常状态时，回调

    const struct dev_pm_ops *pm;  //电源管理相关

    const struct iommu_ops *iommu_ops;

    struct subsys_private *p;         //子系统私有类型
    struct lock_class_key lock_key;
};

struct subsys_private {
    struct kset subsys;          //总线kset，scsi子系统对应/sys/bus/scsi
    struct kset *devices_kset;   //设备kset， scsi子系统对应/sys/bus/scsi/devices
    struct list_head interfaces; //总线的接口链表
    struct mutex mutex;          

    struct kset *drivers_kset;   //驱动kset， scsi子系统对应/sys/bus/scsi/drivers
    struct klist klist_devices;  //总线的设备链表
    struct klist klist_drivers;  //总线的驱动链表
    struct blocking_notifier_head bus_notifier; //子系统变化时，需要通知的链表
    unsigned int drivers_autoprobe:1;  //是否允许设备或驱动加载时，自动探测
    struct bus_type *bus;        //指向总线类型

    struct kset glue_dirs;
    struct class *class;         //指向总线类型的类
};

从上面的两个结构可以看到，bus_type包含的主要是实现子系统应该具体关注的比如name，一组回调函数。而subsys_private结构主要是设备驱动模型中的关系的表达，如字段subsys的类型是kset，描述该子系统在sysfs中的表达；klist_devices和klist_drivers分别是设备链表和驱动链表，用于管理总线类型的所有设备和驱动。之后仍然会遇到xxx_private的结构，以这种方式命名的结构，都是给设备驱动模型核心使用的，业务子系统无需也不能使用。

总线类型注册/反注册

实现子系统的第一步就是创建bus_type，并将其注册到系统，此时需要调用bus_register:

/**
 * bus_register - register a driver-core subsystem
 * @bus: bus to register
 *
 * Once we have that, we register the bus with the kobject
 * infrastructure, then register the children subsystems it has:
 * the devices and drivers that belong to the subsystem.
 */
int bus_register(struct bus_type *bus)
{
    int retval;
    struct subsys_private *priv;
    struct lock_class_key *key = &bus->lock_key;
    //分配总线类型私有数据空间
    priv = kzalloc(sizeof(struct subsys_private), GFP_KERNEL);
    if (!priv)
        return -ENOMEM;

    priv->bus = bus; //关联bus_type和subsys_private
    bus->p = priv;

    BLOCKING_INIT_NOTIFIER_HEAD(&priv->bus_notifier);
    //设置总线类型名称到kobject中，在sysfs中显示
    retval = kobject_set_name(&priv->subsys.kobj, "%s", bus->name);
    if (retval)
        goto out;

    priv->subsys.kobj.kset = bus_kset;
    priv->subsys.kobj.ktype = &bus_ktype;
    priv->drivers_autoprobe = 1;    //开启自动探测

    retval = kset_register(&priv->subsys);  //将总线类型添加到设备模型中
    if (retval)
        goto out;

    retval = bus_create_file(bus, &bus_attr_uevent); //创建uevent属性文件
    if (retval)
        goto bus_uevent_fail;

    priv->devices_kset = kset_create_and_add("devices", NULL,  //创建devices目录
                         &priv->subsys.kobj);
    if (!priv->devices_kset) {
        retval = -ENOMEM;
        goto bus_devices_fail;
    }

    priv->drivers_kset = kset_create_and_add("drivers", NULL,  //创建drivers目录
                         &priv->subsys.kobj);
    if (!priv->drivers_kset) {
        retval = -ENOMEM;
        goto bus_drivers_fail;
    }
    //初始化链表和锁
    INIT_LIST_HEAD(&priv->interfaces);
    __mutex_init(&priv->mutex, "subsys mutex", key);
    klist_init(&priv->klist_devices, klist_devices_get, klist_devices_put);
    klist_init(&priv->klist_drivers, NULL, NULL);

    retval = add_probe_files(bus); //在sysfs中添加探测文件drivers_autoprobe和drivers_probe
    if (retval)
        goto bus_probe_files_fail;

    retval = bus_add_groups(bus, bus->bus_groups); //添加总线类型的属性文件
    if (retval)
        goto bus_groups_fail;

    pr_debug("bus: '%s': registered\n", bus->name);
    return 0;
    //失败回滚操作
bus_groups_fail:
    remove_probe_files(bus);
bus_probe_files_fail:
    kset_unregister(bus->p->drivers_kset);
bus_drivers_fail:
    kset_unregister(bus->p->devices_kset);
bus_devices_fail:
    bus_remove_file(bus, &bus_attr_uevent);
bus_uevent_fail:
    kset_unregister(&bus->p->subsys);
out:
    kfree(bus->p);
    bus->p = NULL;
    return retval;
}
EXPORT_SYMBOL_GPL(bus_register);

注册总线类型后，便可以在系统看到：

root@ubuntu16:~# ls /sys/bus/scsi -l
total 0
drwxr-xr-x 2 root root    0 Sep  5 16:01 devices
drwxr-xr-x 4 root root    0 Sep  2 09:44 drivers
-rw-r--r-- 1 root root 4096 Sep  5 11:29 drivers_autoprobe
--w------- 1 root root 4096 Sep  5 11:29 drivers_probe
--w------- 1 root root 4096 Sep  2 09:44 uevent
root@ubuntu16:~#

当从系统中注销子系统时，需要调用bus_unregister，完成总线类型的反注册：

/**
 * bus_unregister - remove a bus from the system
 * @bus: bus.
 *
 * Unregister the child subsystems and the bus itself.
 * Finally, we call bus_put() to release the refcount
 */
void bus_unregister(struct bus_type *bus)
{
    pr_debug("bus: '%s': unregistering\n", bus->name);
    if (bus->dev_root)
        device_unregister(bus->dev_root);     //删除根设备
    bus_remove_groups(bus, bus->bus_groups);  //删除总线的属性文件
    remove_probe_files(bus);                  //删除探测文件drivers_autoprobe和drivers_probe
    kset_unregister(bus->p->drivers_kset);    //删除drivers目录
    kset_unregister(bus->p->devices_kset);    //删除devices目录
    bus_remove_file(bus, &bus_attr_uevent);   //删除uevent文件
    kset_unregister(&bus->p->subsys);         //删除总线目录
}
EXPORT_SYMBOL_GPL(bus_unregister);

设备

设备的数据结构

struct device {
    struct device       *parent;  //指向父设备，eg.HBA

    struct device_private   *p;   //设备私有指针

    struct kobject kobj;          //内嵌kobject
    const char      *init_name; /* initial name of the device */
    const struct device_type *type;  //设备类型，抽象出来的域和方法

    struct mutex        mutex;  /* mutex to synchronize calls to its driver */

    struct bus_type *bus;       /* type of bus device is on; devive归属的bus */
    struct device_driver *driver;   /* which driver has allocated this device */
    void        *platform_data; /* Platform specific data, device core doesn't touch it */
    void        *driver_data;   /* Driver data, set and get with dev_set/get_drvdata */
    struct dev_pm_info  power;
    struct dev_pm_domain    *pm_domain;

#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
    struct irq_domain   *msi_domain;
#endif
#ifdef CONFIG_PINCTRL
    struct dev_pin_info *pins;
#endif
#ifdef CONFIG_GENERIC_MSI_IRQ
    struct list_head    msi_list;
#endif

#ifdef CONFIG_NUMA
    int     numa_node;  /* NUMA node this device is close to */
#endif
    u64     *dma_mask;  /* dma mask (if dma'able device) */
    u64     coherent_dma_mask;/* Like dma_mask, but for
                         alloc_coherent mappings as
                         not all hardware supports
                         64 bit addresses for consistent
                         allocations such descriptors. */
    unsigned long   dma_pfn_offset;

    struct device_dma_parameters *dma_parms;

    struct list_head    dma_pools;  /* dma pools (if dma'ble) */

    struct dma_coherent_mem *dma_mem; /* internal for coherent mem override */
#ifdef CONFIG_DMA_CMA
    struct cma *cma_area;       /* contiguous memory area for dma allocations */
#endif
    /* arch specific additions */
    struct dev_archdata archdata;

    struct device_node  *of_node; /* associated device tree node */
    struct fwnode_handle    *fwnode; /* firmware device node */

    dev_t           devt;   /* dev_t, creates the sysfs "dev"； 设备号 */
    u32         id; /* device instance */

    spinlock_t      devres_lock;
    struct list_head    devres_head; //设备资源链表头

    struct klist_node   knode_class; //链入类的设备链表
    struct class        *class;      //指向链入的类
    const struct attribute_group **groups;  /* optional groups 设备特有的属性 */

    void    (*release)(struct device *dev);  //设备是否回调
    struct iommu_group  *iommu_group;

    bool            offline_disabled:1;
    bool            offline:1;
};

struct device_private {
    struct klist klist_children;     //子设备链表
    struct klist_node knode_parent;  //链入父设备的children链表
    struct klist_node knode_driver;  //链入驱动的设备链表中
    struct klist_node knode_bus;     //链入总线的设备链表
    struct list_head deferred_probe; //链入延迟探测链表
    struct device *device;           //指向关联的device
};

struct device_type {
    const char *name;  //设备类型的名称
    const struct attribute_group **groups;  //设备的公有属性组
    int (*uevent)(struct device *dev, struct kobj_uevent_env *env); //发送事件前调用，用于设置事件环境变量
    char *(*devnode)(struct device *dev, umode_t *mode, //在创建设备时，提供名字线索
             kuid_t *uid, kgid_t *gid);
    void (*release)(struct device *dev);    //设备释放时回调

    const struct dev_pm_ops *pm;
};

在设备驱动模型中，device结构有bus域，指向device所属的总线类型；class域指向device所属的唯一的类；driver域指向设备所绑定的驱动。与内核对象一样，设备也被组织层层次结构，通过parent指向父设备。

device_private结构由设备驱动模型处理，维护和其他结构之间的内部关系。device_type结构定义设备公有的属性和方法。

设备的注册与反注册

当设备被发现后，需要将设备注册到系统，需要调用device_register函数：

/**
 * device_register - register a device with the system.
 * @dev: pointer to the device structure
 *
 * This happens in two clean steps - initialize the device
 * and add it to the system. The two steps can be called
 * separately, but this is the easiest and most common.
 * I.e. you should only call the two helpers separately if
 * have a clearly defined need to use and refcount the device
 * before it is added to the hierarchy.
 *
 * For more information, see the kerneldoc for device_initialize()
 * and device_add().
 *
 * NOTE: _Never_ directly free @dev after calling this function, even
 * if it returned an error! Always use put_device() to give up the
 * reference initialized in this function instead.
 */
int device_register(struct device *dev)
{
    device_initialize(dev);  //初始化device结构
    return device_add(dev);  //将设备添加到系统
}
EXPORT_SYMBOL_GPL(device_register);


void device_initialize(struct device *dev)
{
    dev->kobj.kset = devices_kset;             // /sys/devices/
    kobject_init(&dev->kobj, &device_ktype);   // device的类型为device_ktype
    INIT_LIST_HEAD(&dev->dma_pools);
    mutex_init(&dev->mutex);
    lockdep_set_novalidate_class(&dev->mutex);
    spin_lock_init(&dev->devres_lock);
    INIT_LIST_HEAD(&dev->devres_head);
    device_pm_init(dev);
    set_dev_node(dev, -1);
#ifdef CONFIG_GENERIC_MSI_IRQ
    INIT_LIST_HEAD(&dev->msi_list);
#endif
}
EXPORT_SYMBOL_GPL(device_initialize);

device_register函数调用device_initialize对device结构进行初始化，调用device_add函数完成设备添加到系统。

int device_add(struct device *dev)
{
    struct device *parent = NULL;
    struct kobject *kobj;
    struct class_interface *class_intf;
    int error = -EINVAL;

    dev = get_device(dev);
    if (!dev)
        goto done;

    if (!dev->p) {  //如果device没有设置devcie_private，在这里分配并初始化
        error = device_private_init(dev);
        if (error)
            goto done;
    }

    /*
     * for statically allocated devices, which should all be converted
     * some day, we need to initialize the name. We prevent reading back
     * the name, and force the use of dev_name()
     */
    if (dev->init_name) {
        dev_set_name(dev, "%s", dev->init_name); //设置device的kobject名字
        dev->init_name = NULL;
    }

    /* subsystems can specify simple device enumeration */
    if (!dev_name(dev) && dev->bus && dev->bus->dev_name) //如果device没有设置init_name, 则使用bus的dev_name和设备id生成
        dev_set_name(dev, "%s%u", dev->bus->dev_name, dev->id);

    if (!dev_name(dev)) {
        error = -EINVAL;
        goto name_error;
    }

    pr_debug("device: '%s': %s\n", dev_name(dev), __func__);

    parent = get_device(dev->parent);
    kobj = get_device_parent(dev, parent);
    if (kobj)
        dev->kobj.parent = kobj;  //设置device的kobject的parent字段

    /* use parent numa_node */
    if (parent && (dev_to_node(dev) == NUMA_NO_NODE))
        set_dev_node(dev, dev_to_node(parent));

    /* first, register with generic layer. */
    /* we require the name to be set before, and pass NULL */
    error = kobject_add(&dev->kobj, dev->kobj.parent, NULL); //将device添加到parent的目录中
    if (error)
        goto Error;

    /* notify platform of device entry */
    if (platform_notify)
        platform_notify(dev);

    error = device_create_file(dev, &dev_attr_uevent); //在设备目录下创建uevent文件
    if (error)
        goto attrError;

    error = device_add_class_symlinks(dev); //为设备创建和类相关的符号链接
    if (error)
        goto SymlinkError;
    error = device_add_attrs(dev); //为设备的默认属性添加对应的文件
    if (error)
        goto AttrsError;
    error = bus_add_device(dev);  //将device添加到bus_type
    if (error)
        goto BusError;
    error = dpm_sysfs_add(dev);
    if (error)
        goto DPMError;
    device_pm_add(dev);

    if (MAJOR(dev->devt)) {
        error = device_create_file(dev, &dev_attr_dev); //在设备目录下创建dev属性对应文件，用于保存设备号
        if (error)
            goto DevAttrError;

        error = device_create_sys_dev_entry(dev); //在/sys/block和/sys/char创建一个到设备所在目录的符号链接
        if (error)
            goto SysEntryError;

        devtmpfs_create_node(dev); //在/dev下创建设备文件
    }

    /* Notify clients of device addition.  This call must come
     * after dpm_sysfs_add() and before kobject_uevent().
     */
    if (dev->bus)
        blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
                         BUS_NOTIFY_ADD_DEVICE, dev);

    kobject_uevent(&dev->kobj, KOBJ_ADD); //发送设备ADD事件
    bus_probe_device(dev);  //尝试将device绑定到device_driver
    if (parent)  //如果指定了parent，将设备添加到parent的孩子链表中
        klist_add_tail(&dev->p->knode_parent,
                   &parent->p->klist_children);

    if (dev->class) {  //如果设置了class，将设备添加到类的设备链表
        mutex_lock(&dev->class->p->mutex);
        /* tie the class to the device */
        klist_add_tail(&dev->knode_class,
                   &dev->class->p->klist_devices);

        /* notify any interfaces that the device is here */
        list_for_each_entry(class_intf,  //调用device所属的class中所有class_interface的add_dev
                    &dev->class->p->interfaces, node)
            if (class_intf->add_dev)
                class_intf->add_dev(dev, class_intf);
        mutex_unlock(&dev->class->p->mutex);
    }
done:
    put_device(dev);
    return error;
 SysEntryError:
    if (MAJOR(dev->devt))
        device_remove_file(dev, &dev_attr_dev);
 DevAttrError:
    device_pm_remove(dev);
    dpm_sysfs_remove(dev);
 DPMError:
    bus_remove_device(dev);
 BusError:
    device_remove_attrs(dev);
 AttrsError:
    device_remove_class_symlinks(dev);
 SymlinkError:
    device_remove_file(dev, &dev_attr_uevent);
 attrError:
    kobject_uevent(&dev->kobj, KOBJ_REMOVE);
    kobject_del(&dev->kobj);
 Error:
    cleanup_device_parent(dev);
    put_device(parent);
name_error:
    kfree(dev->p);
    dev->p = NULL;
    goto done;
}
EXPORT_SYMBOL_GPL(device_add);

设备添加到系统主要流程都在device_add函数实现，上面代码的注释基本把主要函数的作用进行了描述。值得关注的一个函数便是bus_probe_device，该函数完成将设备绑定到驱动的动作。

void bus_probe_device(struct device *dev)
{
    struct bus_type *bus = dev->bus;
    struct subsys_interface *sif;

    if (!bus)
        return;

    if (bus->p->drivers_autoprobe) //如果bus允许自动探测
        device_initial_probe(dev); //主要功能

    mutex_lock(&bus->p->mutex);
    list_for_each_entry(sif, &bus->p->interfaces, node) //将设备绑定到接口
        if (sif->add_dev)
            sif->add_dev(dev, sif);
    mutex_unlock(&bus->p->mutex);
}

void device_initial_probe(struct device *dev)
{
    __device_attach(dev, true);
}

static int __device_attach(struct device *dev, bool allow_async)
{
    int ret = 0;

    device_lock(dev);
    if (dev->driver) {  //指定了device所要绑定的driver
        if (klist_node_attached(&dev->p->knode_driver)) { //检查knode_driver是否绑定到链表
            ret = 1;
            goto out_unlock;
        }
        ret = device_bind_driver(dev); //绑定，修改相应链表
        if (ret == 0)
            ret = 1;
        else {
            dev->driver = NULL;
            ret = 0;
        }
    } else {  //没有指定device要绑定的driver
        struct device_attach_data data = {
            .dev = dev,
            .check_async = allow_async,
            .want_async = false,
        };

        if (dev->parent)
            pm_runtime_get_sync(dev->parent);
        //遍历bus中所有驱动，尝试attach
        ret = bus_for_each_drv(dev->bus, NULL, &data,
                    __device_attach_driver);
        if (!ret && allow_async && data.have_async) {
            /*
             * If we could not find appropriate driver
             * synchronously and we are allowed to do
             * async probes and there are drivers that
             * want to probe asynchronously, we'll
             * try them.
             */
            dev_dbg(dev, "scheduling asynchronous probe\n");
            get_device(dev);
            async_schedule(__device_attach_async_helper, dev);
        } else {
            pm_request_idle(dev);
        }

        if (dev->parent)
            pm_runtime_put(dev->parent);
    }
out_unlock:
    device_unlock(dev);
    return ret;
}

通过上面3个函数的追踪，__device_attach函数遍历bus所有的驱动，尝试执行attach，具体调用__device_attach_driver函数。

static int __device_attach_driver(struct device_driver *drv, void *_data)
{
    struct device_attach_data *data = _data;
    struct device *dev = data->dev;
    bool async_allowed;

    /*
     * Check if device has already been claimed. This may
     * happen with driver loading, device discovery/registration,
     * and deferred probe processing happens all at once with
     * multiple threads.
     */
    if (dev->driver) 
        return -EBUSY;

    if (!driver_match_device(drv, dev))  //调用bus的match函数，测试是否匹配
        return 0;
    //进一步probe设备，需要设备已经注册
    async_allowed = driver_allows_async_probing(drv);

    if (async_allowed)
        data->have_async = true;
    //如果允许异步探测，则先返回
    if (data->check_async && async_allowed != data->want_async)
        return 0;

    return driver_probe_device(drv, dev);
}

int driver_probe_device(struct device_driver *drv, struct device *dev)
{
    int ret = 0;

    if (!device_is_registered(dev)) //检查device是否register
        return -ENODEV;

    pr_debug("bus: '%s': %s: matched device %s with driver %s\n",
         drv->bus->name, __func__, dev_name(dev), drv->name);

    if (dev->parent)
        pm_runtime_get_sync(dev->parent);

    pm_runtime_barrier(dev);
    ret = really_probe(dev, drv); //真正执行探测
    pm_request_idle(dev);

    if (dev->parent)
        pm_runtime_put(dev->parent);

    return ret;
}

从上面两个函数来看，真正执行probe的函数是really_probe。

//返回1表示成功，返回0表示中间步骤出现异常，已回滚所有操作。
static int really_probe(struct device *dev, struct device_driver *drv)
{
    int ret = 0;
    int local_trigger_count = atomic_read(&deferred_trigger_count);

    atomic_inc(&probe_count);
    pr_debug("bus: '%s': %s: probing driver %s with device %s\n",
         drv->bus->name, __func__, drv->name, dev_name(dev));
    WARN_ON(!list_empty(&dev->devres_head));

    dev->driver = drv; //将设备的driver指向当前驱动

    /* If using pinctrl, bind pins now before probing */
    ret = pinctrl_bind_pins(dev);
    if (ret)
        goto probe_failed;

    if (driver_sysfs_add(dev)) {  //在sysfs驱动目录中创建指向设备的符号链接，同时在设备目录中创建指向驱动的符号链接
        printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n",
            __func__, dev_name(dev));
        goto probe_failed;
    }

    if (dev->pm_domain && dev->pm_domain->activate) {
        ret = dev->pm_domain->activate(dev);
        if (ret)
            goto probe_failed;
    }

    /*
     * Ensure devices are listed in devices_kset in correct order
     * It's important to move Dev to the end of devices_kset before
     * calling .probe, because it could be recursive and parent Dev
     * should always go first
     */
    devices_kset_move_last(dev);

    if (dev->bus->probe) {
        ret = dev->bus->probe(dev); //优先调用bus_type中的probe方法
        if (ret)
            goto probe_failed;
    } else if (drv->probe) {
        ret = drv->probe(dev);  //其次，调用driver中的probe方法
        if (ret)
            goto probe_failed;
    }

    pinctrl_init_done(dev);

    if (dev->pm_domain && dev->pm_domain->sync)
        dev->pm_domain->sync(dev);

    driver_bound(dev); //将设备链入驱动的设备链表
    ret = 1;
    pr_debug("bus: '%s': %s: bound device %s to driver %s\n",
         drv->bus->name, __func__, dev_name(dev), drv->name);
    goto done;

probe_failed:  //探测失败, 回滚操作
    devres_release_all(dev);
    driver_sysfs_remove(dev);
    dev->driver = NULL;
    dev_set_drvdata(dev, NULL);
    if (dev->pm_domain && dev->pm_domain->dismiss)
        dev->pm_domain->dismiss(dev);

    switch (ret) {
    case -EPROBE_DEFER:
        /* Driver requested deferred probing */
        dev_dbg(dev, "Driver %s requests probe deferral\n", drv->name);
        driver_deferred_probe_add(dev);
        /* Did a trigger occur while probing? Need to re-trigger if yes */
        if (local_trigger_count != atomic_read(&deferred_trigger_count))
            driver_deferred_probe_trigger();
        break;
    case -ENODEV:
    case -ENXIO:
        pr_debug("%s: probe of %s rejects match %d\n",
             drv->name, dev_name(dev), ret);
        break;
    default:
        /* driver matched but the probe failed */
        printk(KERN_WARNING
               "%s: probe of %s failed with error %d\n",
               drv->name, dev_name(dev), ret);
    }
    /*
     * Ignore errors returned by ->probe so that the next driver can try
     * its luck.
     */
    ret = 0;
done:
    atomic_dec(&probe_count);
    wake_up(&probe_waitqueue);
    return ret;
}

到此，设备添加到系统的主要流程便基本清楚，不再往下跟踪。

驱动

驱动数据结构

struct device_driver {
    const char      *name;     //driver名称
    struct bus_type     *bus;  //driver所属的bus_type

    struct module       *owner;
    const char      *mod_name;  /* used for built-in modules */

    bool suppress_bind_attrs;   /* disables bind/unbind via sysfs */
    enum probe_type probe_type;

    const struct of_device_id   *of_match_table;
    const struct acpi_device_id *acpi_match_table;

    int (*probe) (struct device *dev);  //在device绑定到driver之前，对device进行初始化
    int (*remove) (struct device *dev); //在device解绑到driver时，回调
    void (*shutdown) (struct device *dev);
    int (*suspend) (struct device *dev, pm_message_t state);
    int (*resume) (struct device *dev);
    const struct attribute_group **groups; //driver的属性

    const struct dev_pm_ops *pm; //电源相关

    struct driver_private *p;  //driver私有结构
};

struct driver_private {
    struct kobject kobj;
    struct klist klist_devices;   //driver所支持的device链表
    struct klist_node knode_bus;  //链入bus_type的驱动链表中
    struct module_kobject *mkobj;
    struct device_driver *driver;  //指向driver
};

device_driver结构中，bus域指向驱动所属的总线类型，knode_bus域用于链入总线类型的驱动链表。driver_private结构中的klist_devices域用于链接所有绑定到本驱动的设备。

驱动注册与反注册

驱动在加载时，需要将其注册到总线类型，调用driver_register实现：

int driver_register(struct device_driver *drv)
{
    int ret;
    struct device_driver *other;

    BUG_ON(!drv->bus->p); //确保bus已经注册到驱动模型中
    //如果bus_type和driver都实现了同一个回调，优先使用bus_type的回调函数，打印告警信息
    if ((drv->bus->probe && drv->probe) ||
        (drv->bus->remove && drv->remove) ||
        (drv->bus->shutdown && drv->shutdown))
        printk(KERN_WARNING "Driver '%s' needs updating - please use "
            "bus_type methods\n", drv->name);

    other = driver_find(drv->name, drv->bus); //根据名字查找驱动
    if (other) {
        printk(KERN_ERR "Error: Driver '%s' is already registered, "
            "aborting...\n", drv->name);
        return -EBUSY;
    }

    ret = bus_add_driver(drv); //将driver添加到bus
    if (ret)
        return ret;
    ret = driver_add_groups(drv, drv->groups); //创建driver属性文件
    if (ret) {
        bus_remove_driver(drv);
        return ret;
    }
    kobject_uevent(&drv->p->kobj, KOBJ_ADD); //发送ADD事件到用户空间

    return ret;
}
EXPORT_SYMBOL_GPL(driver_register);

添加driver到bus_type，由bus_add_driver实现：

int bus_add_driver(struct device_driver *drv)
{
    struct bus_type *bus;
    struct driver_private *priv;
    int error = 0;

    bus = bus_get(drv->bus);
    if (!bus)
        return -EINVAL;

    pr_debug("bus: '%s': add driver %s\n", bus->name, drv->name);

    priv = kzalloc(sizeof(*priv), GFP_KERNEL);  //分配driver_private结构空间
    if (!priv) {
        error = -ENOMEM;
        goto out_put_bus;
    }
    klist_init(&priv->klist_devices, NULL, NULL); //初始化driver设备链表
    priv->driver = drv; //关联device_driver和driver_private
    drv->p = priv;
    priv->kobj.kset = bus->p->drivers_kset; //driver_private中的kobj的kset域指向subsys中的drivers_kset
    error = kobject_init_and_add(&priv->kobj, &driver_ktype, NULL,  //添加driver到sysfs
                     "%s", drv->name);
    if (error)
        goto out_unregister;

    klist_add_tail(&priv->knode_bus, &bus->p->klist_drivers); //添加driver到bus的驱动链表中
    if (drv->bus->p->drivers_autoprobe) {  //自动探测
        if (driver_allows_async_probing(drv)) {  //允许异步执行probe
            pr_debug("bus: '%s': probing driver %s asynchronously\n",
                drv->bus->name, drv->name);
            async_schedule(driver_attach_async, drv); //异步probe
        } else {
            error = driver_attach(drv);  //同步probe
            if (error)
                goto out_unregister;
        }
    }
    module_add_driver(drv->owner, drv);  //驱动实现的模块

    error = driver_create_file(drv, &driver_attr_uevent);  //在driver中添加uevent属性文件
    if (error) {
        printk(KERN_ERR "%s: uevent attr (%s) failed\n",
            __func__, drv->name);
    }
    error = driver_add_groups(drv, bus->drv_groups);  //添加driver的属性文件
    if (error) {
        /* How the hell do we get out of this pickle? Give up */
        printk(KERN_ERR "%s: driver_create_groups(%s) failed\n",
            __func__, drv->name);
    }

    if (!drv->suppress_bind_attrs) {
        error = add_bind_files(drv);  //在driver目录添加的bind和unbind两个属性文件
        if (error) {
            /* Ditto */
            printk(KERN_ERR "%s: add_bind_files(%s) failed\n",
                __func__, drv->name);
        }
    }

    return 0;

out_unregister:
    kobject_put(&priv->kobj);
    kfree(drv->p);
    drv->p = NULL;
out_put_bus:
    bus_put(bus);
    return error;
}

bus_add_driver函数完成驱动添加到总线类型，当驱动添加完成后，如果总线类型设置了允许自动探测标志drivers_autoprobe，便可以根据是否允许异步探测调用driver_attach_async或driver_attach，driver_attach_async也是调用driver_attach:

int driver_attach(struct device_driver *drv)
{
    return bus_for_each_dev(drv->bus, NULL, drv, __driver_attach);
}
EXPORT_SYMBOL_GPL(driver_attach);

static int __driver_attach(struct device *dev, void *data)
{
    struct device_driver *drv = data;

    /*
     * Lock device and try to bind to it. We drop the error
     * here and always return 0, because we need to keep trying
     * to bind to devices and some drivers will return an error
     * simply if it didn't support the device.
     *
     * driver_probe_device() will spit a warning if there
     * is an error.
     */

    if (!driver_match_device(drv, dev)) //调用bus_type.match
        return 0;

    if (dev->parent)    /* Needed for USB */
        device_lock(dev->parent);
    device_lock(dev);
    if (!dev->driver)
        driver_probe_device(drv, dev); //完成probe的主要函数
    device_unlock(dev);
    if (dev->parent)
        device_unlock(dev->parent);

    return 0;
}

int driver_probe_device(struct device_driver *drv, struct device *dev)
{
    int ret = 0;

    if (!device_is_registered(dev)) //检查device是否register
        return -ENODEV;

    pr_debug("bus: '%s': %s: matched device %s with driver %s\n",
         drv->bus->name, __func__, dev_name(dev), drv->name);

    if (dev->parent)
        pm_runtime_get_sync(dev->parent);

    pm_runtime_barrier(dev);
    ret = really_probe(dev, drv); //真正执行探测
    pm_request_idle(dev);

    if (dev->parent)
        pm_runtime_put(dev->parent);

    return ret;
}

根据上面3个函数，最终仍然是调用前面描述过的really_probe函数完成最后的探测。

到这里驱动注册完成，结合之前的设备注册流程，无论是驱动注册或是设备注册，只要总线类型设置了自动探测标志位，这两个流程都会执行探测。所以设备发现与驱动的加载顺序已经不再重要，也是通过这种双向探测方式，Linux内核支持设备的热拔插机制。

驱动卸载时，需要调用driver_unregister函数，使driver脱离总线类型：

void driver_unregister(struct device_driver *drv)
{
    if (!drv || !drv->p) {
        WARN(1, "Unexpected driver unregister!\n");
        return;
    }
    driver_remove_groups(drv, drv->groups); //删除驱动的属性文件
    bus_remove_driver(drv);                 //从总线类型中移除驱动
}
EXPORT_SYMBOL_GPL(driver_unregister);

void bus_remove_driver(struct device_driver *drv)
{
    if (!drv->bus)
        return;

    if (!drv->suppress_bind_attrs)
        remove_bind_files(drv);   //删除驱动目录下bind和unbind文件
    driver_remove_groups(drv, drv->bus->drv_groups); //删除总线类型的驱动属性文件
    driver_remove_file(drv, &driver_attr_uevent);    //删除驱动目录下uevent文件
    klist_remove(&drv->p->knode_bus); //从总线类型的驱动链表中移除驱动
    pr_debug("bus: '%s': remove driver %s\n", drv->bus->name, drv->name);
    driver_detach(drv);  //驱动与所有绑定的设备进行解绑
    module_remove_driver(drv);  //驱动实现的模块
    kobject_put(&drv->p->kobj); //减少引用计数
    bus_put(drv->bus);
}

类

类数据结构

struct class {
    const char      *name;       //类名称
    struct module       *owner;  //指向实现这个类的模块的指针

    struct class_attribute      *class_attrs;     //类公共属性
    const struct attribute_group    **dev_groups; //归属与该类的设备的默认属性
    struct kobject          *dev_kobj;            //类链入sysfs的kobject

    int (*dev_uevent)(struct device *dev, struct kobj_uevent_env *env); //发送事件前，设置类的特定环境变量
    char *(*devnode)(struct device *dev, umode_t *mode); //创建设备时，返回设备名称

    void (*class_release)(struct class *class); //类释放时回调
    void (*dev_release)(struct device *dev);    //设备释放时回调

    int (*suspend)(struct device *dev, pm_message_t state); //设备进入睡眠状态时，回调
    int (*resume)(struct device *dev);                      //设备被唤醒时，回调

    const struct kobj_ns_type_operations *ns_type;  //sysfs支持命名空间
    const void *(*namespace)(struct device *dev);   //返回设备所在的命名空间

    const struct dev_pm_ops *pm;  //电源相关

    struct subsys_private *p;     //类所属的子系统私有数据结构
};

类的私有数据类型与总线类型的私有数据类型都是subsys_private，这里将不再重复描述。

类注册与反注册

子系统需要使用类时，需要调用class_register函数向总线类型注册类：

#define class_register(class)           \
({                      \
    static struct lock_class_key __key; \
    __class_register(class, &__key);    \
})

int __class_register(struct class *cls, struct lock_class_key *key)
{
    struct subsys_private *cp;
    int error;

    pr_debug("device class '%s': registering\n", cls->name);

    cp = kzalloc(sizeof(*cp), GFP_KERNEL); //分配私有数据空间
    if (!cp)
        return -ENOMEM;
    klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put); //初始化该class的device链表
    INIT_LIST_HEAD(&cp->interfaces);  //初始化接口链表
    kset_init(&cp->glue_dirs);
    __mutex_init(&cp->mutex, "subsys mutex", key);
    error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name); //将在/sys/class/目录下显示该名称
    if (error) {
        kfree(cp);
        return error;
    }

    /* set the default /sys/dev directory for devices of this class */
    if (!cls->dev_kobj)
        cls->dev_kobj = sysfs_dev_char_kobj;

#if defined(CONFIG_BLOCK)
    /* let the block class directory show up in the root of sysfs */
    if (!sysfs_deprecated || cls != &block_class)
        cp->subsys.kobj.kset = class_kset;
#else
    cp->subsys.kobj.kset = class_kset;  // 全局变量class_kset指的是 /sys/class/
#endif
    cp->subsys.kobj.ktype = &class_ktype;
    cp->class = cls;  //class与subsys_private关联
    cls->p = cp;

    error = kset_register(&cp->subsys);  //在/sys/class/目录下创建该类对应的目录
    if (error) {
        kfree(cp);
        return error;
    }
    error = add_class_attrs(class_get(cls)); //在/sys/class/xxx/目录下创建类属性文件
    class_put(cls);
    return error;
}
EXPORT_SYMBOL_GPL(__class_register);

类的注册比较简单，注释已经比较详细。当子系统需要卸载类时，需要调用class_register函数：

void class_unregister(struct class *cls)
{
    pr_debug("device class '%s': unregistering\n", cls->name);
    remove_class_attrs(cls);            //删除/sys/class/xxx/目录下的类属性文件
    kset_unregister(&cls->p->subsys);   //删除/sys/class/目录
}

接口

接口数据结构

struct class_interface {
    struct list_head    node;    //链入class中
    struct class        *class;  //指向所属的class 
    //在接口被添加或者设备被添加到接口所在的类时，从接口中添加或删除设备
    int (*add_dev)      (struct device *, struct class_interface *);
    void (*remove_dev)  (struct device *, struct class_interface *);
};

接口注册与反注册

向类中注册接口，需要调用class_interface_register函数完成：

int class_interface_register(struct class_interface *class_intf)
{
    struct class *parent;
    struct class_dev_iter iter;
    struct device *dev;

    if (!class_intf || !class_intf->class)  //确保class和class_interface都存在
        return -ENODEV;

    parent = class_get(class_intf->class); //增加引用计数，并返回接口所属的class
    if (!parent)
        return -EINVAL;

    mutex_lock(&parent->p->mutex);
    list_add_tail(&class_intf->node, &parent->p->interfaces); //将class_interface添加到class的接口链表
    if (class_intf->add_dev) {  //如果接口设置了add_dev方法，对该class的所有device调用
        class_dev_iter_init(&iter, parent, NULL, NULL);
        while ((dev = class_dev_iter_next(&iter)))
            class_intf->add_dev(dev, class_intf);  //接口方法作用于设备
        class_dev_iter_exit(&iter);
    }
    mutex_unlock(&parent->p->mutex);

    return 0;
}

从类中删除接口，需要调用class_interface_unregister函数完成：

void class_interface_unregister(struct class_interface *class_intf)
{
    struct class *parent = class_intf->class;
    struct class_dev_iter iter;
    struct device *dev;

    if (!parent)
        return;

    mutex_lock(&parent->p->mutex);
    list_del_init(&class_intf->node); //将class_interface从class的接口链表中删除
    if (class_intf->remove_dev) { //如果接口设置了remove_dev方法，对该class的所有device调用
        class_dev_iter_init(&iter, parent, NULL, NULL);
        while ((dev = class_dev_iter_next(&iter)))
            class_intf->remove_dev(dev, class_intf);  //接口方法作用于设备
        class_dev_iter_exit(&iter);
    }
    mutex_unlock(&parent->p->mutex);

    class_put(parent);
}

基于设备驱动模型实现子系统

Linux设备驱动模型已经将每种对象的关系，sysfs的呈现方式已经实现了。实现子系统只需要定义业务自身的总线类型, 设备, 驱动, 类, 接口分别”继承”bus_type, device, driver, class, class_interface。并根据具体业务实现各个结构规定的回调函数。最后调用上述的注册函数添加到系统，便完成子系统的开发。