smmu学习笔记之bus_set_iommu

在arm_smmu_device_probe 中会针对pci bus设定iommu_ops
#ifdef CONFIG_PCI
    if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
        pci_request_acs();
        ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
        if (ret)
            return ret;
    }
#endif
这个if条件显然成立
int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
{
    int err;

    if (bus->iommu_ops != NULL)
        return -EBUSY;

    bus->iommu_ops = ops;

    /* Do IOMMU specific setup for this bus-type */
    err = iommu_bus_init(bus, ops);
    if (err)
        bus->iommu_ops = NULL;

    return err;
}
bus_set_iommu 中将arm_smmu_ops 赋值给bus->iommu_ops，然后调用iommu_bus_init 为bus上的每个device添加group
static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
{
    int err;
    struct notifier_block *nb;
    struct iommu_callback_data cb = {
        .ops = ops,
    };

    nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
    if (!nb)
        return -ENOMEM;

    nb->notifier_call = iommu_bus_notifier;

    err = bus_register_notifier(bus, nb);
    if (err)
        goto out_free;

    err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
    if (err)
        goto out_err;


    return 0;

}
iommu_bus_init中会通过bus_for_each_dev遍历pci上的device，会为每个device调用add_iommu_group
static int add_iommu_group(struct device *dev, void *data)
{
    struct iommu_callback_data *cb = data;
    const struct iommu_ops *ops = cb->ops;
    int ret;

    if (!ops->add_device)
        return 0;

    WARN_ON(dev->iommu_group);

    ret = ops->add_device(dev);
}
add_iommu_group 只是一个wrapper函数最终还是调用ops->add_device，前面我们讲了ops就等于arm_smmu_ops，因此调用
arm_smmu_add_device->iommu_group_get_for_dev
struct iommu_group *iommu_group_get_for_dev(struct device *dev)
{
    const struct iommu_ops *ops = dev->bus->iommu_ops;
    struct iommu_group *group;
    int ret;

    group = iommu_group_get(dev);
    if (group)
        return group;

    group = ERR_PTR(-EINVAL);

    if (ops && ops->device_group)
        group = ops->device_group(dev);

    if (IS_ERR(group))
        return group;

    /*
     * Try to allocate a default domain - needs support from the
     * IOMMU driver.
     */
    if (!group->default_domain) {
        group->default_domain = __iommu_domain_alloc(dev->bus,
                                 IOMMU_DOMAIN_DMA);
        if (!group->domain)
            group->domain = group->default_domain;
    }

    ret = iommu_group_add_device(group, dev);
    if (ret) {
        iommu_group_put(group);
        return ERR_PTR(ret);
    }

    return group;
}
在iommu_group_get_for_dev 中首先通过iommu_group_get得到group，这个时候group肯定是null的，但是ops->device_group 不为null，因此通过group = ops->device_group(dev);赋值，这里的device_group就是arm_smmu_device_group
static struct iommu_group *arm_smmu_device_group(struct device *dev)
{
    struct iommu_group *group;

    /*
     * We don't support devices sharing stream IDs other than PCI RID
     * aliases, since the necessary ID-to-device lookup becomes rather
     * impractical given a potential sparse 32-bit stream ID space.
     */
    if (dev_is_pci(dev))
        group = pci_device_group(dev);
    else
        group = generic_device_group(dev);

    return group;
}
由于我们是pci因此group就是pci_device_group。
回到iommu_group_get_for_dev 中继续调用iommu_group_add_device
int iommu_group_add_device(struct iommu_group *group, struct device *dev)
{
    int ret, i = 0;
    struct iommu_device *device;

    device = kzalloc(sizeof(*device), GFP_KERNEL);
    if (!device)
        return -ENOMEM;

    device->dev = dev;

    ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
###
创建iommu_group的软连接
root@ubuntu:/sys/kernel/iommu_groups/0/devices/0002:80:00.0# ll
total 0
drwxr-xr-x 4 root root    0 Mar  7 16:01 ./
drwxr-xr-x 6 root root    0 Mar  7 16:01 ../
-rw-r--r-- 1 root root 4096 Mar  7 16:01 broken_parity_status
-r--r--r-- 1 root root 4096 Mar  7 16:01 class
-rw-r--r-- 1 root root 4096 Mar  7 16:01 config
-r--r--r-- 1 root root 4096 Mar  7 16:01 consistent_dma_mask_bits
-rw-r--r-- 1 root root 4096 Mar  7 16:01 d3cold_allowed
-r--r--r-- 1 root root 4096 Mar  7 16:01 device
-r--r--r-- 1 root root 4096 Mar  7 16:01 devspec
-r--r--r-- 1 root root 4096 Mar  7 16:01 dma_mask_bits
lrwxrwxrwx 1 root root    0 Mar  7 16:01 driver -> ../../../bus/pci/drivers/pcieport/
-rw-r--r-- 1 root root 4096 Mar  7 16:01 driver_override
-rw-r--r-- 1 root root 4096 Mar  7 16:01 enable
lrwxrwxrwx 1 root root    0 Mar  7 16:01 iommu_group -> ../../../kernel/iommu_groups/0/

###
    device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
rename:
    if (!device->name) {
        sysfs_remove_link(&dev->kobj, "iommu_group");
        kfree(device);
        return -ENOMEM;
    }

    ret = sysfs_create_link_nowarn(group->devices_kobj,
                       &dev->kobj, device->name);
##
创建软连接
root@ubuntu:/sys/kernel/iommu_groups/0/devices# ll
total 0
drwxr-xr-x 2 root root 0 Mar  7 16:41 ./
drwxr-xr-x 3 root root 0 Mar  7 16:41 ../
lrwxrwxrwx 1 root root 0 Mar  7 16:41 0002:80:00.0 -> ../../../../devices/pci0002:80/0002:80:00.0/

##

    kobject_get(group->devices_kobj);

    dev->iommu_group = group;

    iommu_group_create_direct_mappings(group, dev);

    mutex_lock(&group->mutex);
    list_add_tail(&device->list, &group->devices);
    if (group->domain)
        __iommu_attach_device(group->domain, dev);
    mutex_unlock(&group->mutex);

    /* Notify any listeners about change to group. */
    blocking_notifier_call_chain(&group->notifier,
                     IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);

    trace_add_device_to_group(group->id, dev);

    pr_info("Adding device %s to group %d\n", dev_name(dev), group->id);

    return 0;
}

这样就能在开机log中看到[    4.845579] iommu: Adding device 0002:80:00.0 to group 0  字样
通过在iommu_groups 下就能看到当前的device，可见device id是从0开始的
root@ubuntu:/sys/kernel/iommu_groups# ls
0  1