dpdk源码---vfio(zym)

主要想找到从nvme设备 到 vfio驱动的代码路径

[spdk/lib/nvme/nvme_pcie.c]

nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,bool direct_connect) (nvme_tcp,nvme_rdma等都有自己的scan,probe等函数，统一被封装起来,如下)

1 int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
2      bool direct_connect)
3 {
4  NVME_TRANSPORT_CALL(probe_ctx->trid.trtype, ctrlr_scan, (probe_ctx, direct_connect));
5 }

1.nvme_pcie_ctrl_scan:

 1 int
 2 nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
 3              bool direct_connect)
 4 {
 5     .......//若probe_ctx中有指定nvme pci id，has_pci_addr=true
 6     if (enum_ctx.has_pci_addr == false) {
 7         return spdk_pci_enumerate(spdk_pci_nvme_get_driver(),//若没有指定
 8                       pcie_nvme_enum_cb, &enum_ctx);
 9     } else {
10         return spdk_pci_device_attach(spdk_pci_nvme_get_driver(),//指定了话直接attach
11                           pcie_nvme_enum_cb, &enum_ctx, &enum_ctx.pci_addr);
12     }
13 

spdk_pci_enumerate:

 1 int
 2 spdk_pci_enumerate(struct spdk_pci_driver *driver,
 3            spdk_pci_enum_cb enum_cb,
 4            void *enum_ctx)
 5 {
 6     struct spdk_pci_device *dev;
 7     int rc;
 8 
 9     cleanup_pci_devices();
10 
11     pthread_mutex_lock(&g_pci_mutex);
12     TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { //遍历g_pci_devices
13         if (dev->internal.attached ||
14             dev->internal.driver != driver ||
15             dev->internal.pending_removal) {
16             continue;
17         }
18 
19         rc = enum_cb(enum_ctx, dev);// 从g_pci_devices找到我们要的driver，那么调用回调函数进行后续的ctrl_construct工作
20         if (rc == 0) {
21             dev->internal.attached = true;//构建成功，attach设置为true
22         } else if (rc < 0) {
23             pthread_mutex_unlock(&g_pci_mutex);
24             return -1;
25         }
26     }
27     pthread_mutex_unlock(&g_pci_mutex);
28 
29     if (!driver->is_registered) {//若改driver还没注册，那么register到rte_pci_bus.driver_list
30         driver->is_registered = true;
31         rte_pci_register(&driver->driver);
32     }
33 //上面不是找到了吗，下面这些是干啥？为啥要scan和probe？？上面只是放到rte_pci_bus.driver_list了，后面还需做什么？
34     driver->cb_fn = enum_cb;
35     driver->cb_arg = enum_ctx;
36 
37     if (rte_bus_scan() != 0 || rte_bus_probe() != 0) {//若bus scan和probe都不成功
38         driver->cb_arg = NULL;
39         driver->cb_fn = NULL;
40         return -1;
41     }
42 
43     driver->cb_arg = NULL;
44     driver->cb_fn = NULL;
45 
46     cleanup_pci_devices();
47     return 0;
48 }

回调函数pcie_nvme_enum_cb：

主要是执行nvme_ctrlr_probe(&trid, enum_ctx->probe_ctx, pci_dev) 从而构建相应type的ctrl：nvme_transport_ctrlr_construct(trid, &opts, devhandle)，这些不放在这里介绍，会在spdk的源码解读里面分析。

rte_pci_register:

1 /* register a driver */
2 void
3 rte_pci_register(struct rte_pci_driver *driver)
4 {
5     TAILQ_INSERT_TAIL(&rte_pci_bus.driver_list, driver, next);
6     driver->bus = &rte_pci_bus;
7 }//注意这个struct rte_pci_bus rte_pci_bus = { .bus = {  .scan = rte_pci_scan,  .probe = rte_pci_probe,  .find_device = pci_find_device,  .plug = pci_plug,  .unplug = pci_unplug,  .parse = pci_parse,  .dma_map = pci_dma_map,  .dma_unmap = pci_dma_unmap,  .get_iommu_class = rte_pci_get_iommu_class,  .dev_iterate = rte_pci_dev_iterate,  .hot_unplug_handler = pci_hot_unplug_handler,  .sigbus_handler = pci_sigbus_handler, }, .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list), .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list),};

rte_bus_scan()和rte_bus_probe() 对所有bus进行scan和probe，若是pci_bus,执行的是上面注册的

.scan = rte_pci_scan, .probe = rte_pci_probe,

回到 1 中的attach函数：

spdk_pci_device_attach：跟原来看的有些不一样，之前看的版本直接调用的是rte_pci_scan

 1 int
 2 spdk_pci_device_attach(struct spdk_pci_driver *driver,
 3                spdk_pci_enum_cb enum_cb,
 4                void *enum_ctx, struct spdk_pci_addr *pci_address)
 5 {
 6     struct spdk_pci_device *dev;
 7     int rc;
 8     char bdf[32];
 9 
10     spdk_pci_addr_fmt(bdf, sizeof(bdf), pci_address);
11 
12     cleanup_pci_devices();
13 
14     TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) {
15         if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) {
16             break;
17         }
18     }
19 
20     if (dev != NULL && dev->internal.driver == driver) {
21         pthread_mutex_lock(&g_pci_mutex);
22         if (dev->internal.attached || dev->internal.pending_removal) {
23             pthread_mutex_unlock(&g_pci_mutex);
24             return -1;
25         }
26 
27         rc = enum_cb(enum_ctx, dev);//执行construct ctrl等
28         if (rc == 0) {
29             dev->internal.attached = true;
30         }
31         pthread_mutex_unlock(&g_pci_mutex);
32         return rc;
33     }
34 
35     if (!driver->is_registered) {
36         driver->is_registered = true;
37         rte_pci_register(&driver->driver);//register到rte_pci_bus.driver_list
38     }
39 
40     drive