Linux Sysfs - /sys/bus/pci/devices的numa_node 如何获取的

查看pcie device的 numa_node:

cat /sys/bus/pci/devices/0000\:00\:00.0/numa_node
1
cat /sys/bus/pci/devices/0001\:00\:00.0/numa_node
0
 

从代码可以看到， pcie 设备的numa_node 是由所在总线的 numa_node得来的：

//drivers\pci\probe.c

void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
{
    int ret;

    pci_configure_device(dev);

    device_initialize(&dev->dev);
    dev->dev.release = pci_release_dev;

    set_dev_node(&dev->dev, pcibus_to_node(bus));
    dev->dev.dma_mask = &dev->dma_mask;
    dev->dev.dma_parms = &dev->dma_parms;
    dev->dev.coherent_dma_mask = 0xffffffffull;

....

    /* Notifier could use PCI capabilities */
    dev->match_driver = false;
    ret = device_add(&dev->dev);
    WARN_ON(ret < 0);
}

tree /sys/bus/pci/devices/
/sys/bus/pci/devices/
├── 0000:00:00.0 -> ../../../devices/pci0000:00/0000:00:00.0
├── 0000:00:01.0 -> ../../../devices/pci0000:00/0000:00:01.0
├── 0001:00:00.0 -> ../../../devices/pci0001:00/0001:00:00.0
├── 0001:00:01.0 -> ../../../devices/pci0001:00/0001:00:01.0
而设备 0001:00:00.0 是挂在bus pci0001:00下，

而设备 0000:00:00.0 是挂在bus pci0000:00下，那么这俩个总线的numa_node 是多少呢？

再来查找注册 host bridge的代码：

//drivers\pci\probe.c

static int pci_register_host_bridge(struct pci_host_bridge *bridge)
{
    struct device *parent = bridge->dev.parent;
    struct resource_entry *window, *n;
    struct pci_bus *bus, *b;
    resource_size_t offset;
    LIST_HEAD(resources);
    struct resource *res;
    char addr[64], *fmt;
    const char *name;
    int err;

    bus = pci_alloc_bus(NULL);
    if (!bus)
        return -ENOMEM;

    bridge->bus = bus;

    /* Temporarily move resources off the list */
    list_splice_init(&bridge->windows, &resources);
    bus->sysdata = bridge->sysdata;                                //总线的sysdata 跟 bridge的 sysdata一样
    bus->msi = bridge->msi;
    bus->ops = bridge->ops;
    bus->number = bus->busn_res.start = bridge->busnr;
#ifdef CONFIG_PCI_DOMAINS_GENERIC
    bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
#endif

    b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
    if (b) {
        /* Ignore it if we already got here via a different bridge */
        dev_dbg(&b->dev, "bus already known\n");
        err = -EEXIST;
        goto free;
    }

    dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
             bridge->busnr);

    err = pcibios_root_bridge_prepare(bridge);
    if (err)
        goto free;

    err = device_register(&bridge->dev);
    if (err)
        put_device(&bridge->dev);

    bus->bridge = get_device(&bridge->dev);
    device_enable_async_suspend(bus->bridge);
    pci_set_bus_of_node(bus);
    pci_set_bus_msi_domain(bus);

    if (!parent)
        set_dev_node(bus->bridge, pcibus_to_node(bus));  //bus对应的bridge, numa_node也是跟bus保持一致

    bus->dev.class = &pcibus_class;
    bus->dev.parent = bus->bridge;

    dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
    name = dev_name(&bus->dev);

    err = device_register(&bus->dev);
    if (err)
        goto unregister;

 .....

}
 

而下面的函数用来设置bus的 numa_node, 

//arch\arm64\kernel\pci.c

int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
{
    if (!acpi_disabled) {
        struct pci_config_window *cfg = bridge->bus->sysdata;
        struct acpi_device *adev = to_acpi_device(cfg->parent);
        struct device *bus_dev = &bridge->bus->dev;

        ACPI_COMPANION_SET(&bridge->dev, adev);
        set_dev_node(bus_dev, acpi_get_node(acpi_device_handle(adev)));
    }

    return 0;
}

而具体的node 信息是acpi_get_nod()从acpi 获取过来， 看看下面_PXM， 就能猜到，

这个是根据 ACPI  DSDT table 中  PCI bridge 设备定义获取来的， 具体的就得看UEFI ACPI 的定义了

//drivers\acpi\numa.c

static int acpi_get_pxm(acpi_handle h)
{
    unsigned long long pxm;
    acpi_status status;
    acpi_handle handle;
    acpi_handle phandle = h;

    do {
        handle = phandle;
        status = acpi_evaluate_integer(handle, "_PXM", NULL, &pxm);
        if (ACPI_SUCCESS(status))
            return pxm;
        status = acpi_get_parent(handle, &phandle);
    } while (ACPI_SUCCESS(status));
    return -1;
}

int acpi_get_node(acpi_handle handle)
{
    int pxm;

    pxm = acpi_get_pxm(handle);

    return acpi_map_pxm_to_node(pxm);
}

UEFI 定义：

  // PCI3 RCA3
  Device (PCI3) {
    //
    // Hardware ID must be PNP0A08, which maps to a PCIe root complex.
    // Section 6.1.5
    //

    Name (_HID, "PNP0A08")
    Name (_CCA, ONE)

.....

    //
    // Root Port 0 Device within the Root Complex.
    //
    Device (RP0) {
      //
      // Device 0, Function 0.
      //

      Name (_ADR, 0x00000000)
    }

    Method (_PXM, 0, NotSerialized) {
      // Patch by code
      Return(0xFF)
    }
  } // PCI3 RCA3

而具体的 _PXM 是有UEFI 代码 根据 系统numa的 设定获取的。