Linux Topics (4) - PCI Bus Scan from RC

• RC1 : Device ( PCI1 ) {
– _CRS{}
• MEM1 . // MIN= 0x4000000 , LEN= 0x10000000 , TRA = 0x740000000000
• MEM2 : // MIN= 0x740100000000 , , LEN = 0x3FEE0000000 , MAX= 0x77ffdfffffff

3. PCI Bus Scan - acpi_pci_root_add()

acpi_pci_root_add()

pci_acpi_scan_root()

root_ops->release_info = pci_acpi_generic_release_info;

root_ops->prepare_resources = pci_acpi_root_prepare_resources;

acpi_pci_root_create()

|-->ops->prepare_resources(info);

| |-->pci_acpi_root_prepare_resources; //Get resource from ACPI(_CRS..)

|--> pci_acpi_root_add_resources(info);

|-->pci_create_root_bus()

| |-->pci_register_host_bridge()

|-->pci_scan_child_bus(bus);

| pci_scan_child_bus_extend(bus, 0);

| pci_scan_slot(bus, devfn);

| pci_scan_single_device()

| |-->pci_scan_device(bus, devfn); //扫描slot，并创建pci dev

| |-->pci_device_add(dev, bus); //所有设备

|-->pci_assign_unassigned_root_bus_resources()

__pci_bus_assign_resources

pbus_assign_resources_sorted()

__assign_resources_sorted()

assign_requested_resources_sorted

pci_assign_resource()

pci_setup_bridge();

Details:

1. 获取 PCI RC的 segment, 根据 UEFI ACPI Device 汇报的 _SEG

2. 获取 Start bus number: "Check _CRS first, then _BBN.", try_get_root_bridge_busnr(handle, &root->secondary);

3. 获取 _OSC capability ： negotiate_os_control(root, &no_aspm);

4. Scan the Root Bridge: pci_acpi_scan_root()

static int acpi_pci_root_add(struct acpi_device *device, const struct acpi_device_id *not_used)

{

unsigned long long segment, bus;

acpi_status status;

int result;

struct acpi_pci_root *root;

acpi_handle handle = device->handle;

int no_aspm = 0;

bool hotadd = system_state == SYSTEM_RUNNING;

const char *acpi_hid;

root = kzalloc(sizeof(struct acpi_pci_root), GFP_KERNEL);

segment = 0;

status = acpi_evaluate_integer(handle, METHOD_NAME__SEG, NULL, &segment);

/* Check _CRS first, then _BBN. If no _BBN, default to zero. */

root->secondary.flags = IORESOURCE_BUS;

status = try_get_root_bridge_busnr(handle, &root->secondary);

if (ACPI_FAILURE(status)) {

* We need both the start and end of the downstream bus range

* to interpret _CBA (MMCONFIG base address), so it really is supposed to be in _CRS.

If we don't find it there, all we can do is assume [_BBN-0xFF] or [0-0xFF].

root->secondary.end = 0xFF;

dev_warn(&device->dev, FW_BUG "no secondary bus range in _CRS\n");

status = acpi_evaluate_integer(handle, METHOD_NAME__BBN, NULL, &bus);

if (ACPI_SUCCESS(status))

root->secondary.start = bus;

else if (status == AE_NOT_FOUND)

root->secondary.start = 0;

else {

dev_err(&device->dev, "can't evaluate _BBN\n");

result = -ENODEV;

goto end;

}

root->device = device;

root->segment = segment & 0xFFFF;

strcpy(acpi_device_name(device), ACPI_PCI_ROOT_DEVICE_NAME);

strcpy(acpi_device_class(device), ACPI_PCI_ROOT_CLASS);

device->driver_data = root;

pr_info("%s [%s] (domain %04x %pR)\n", acpi_device_name(device), acpi_device_bid(device),

root->segment, &root->secondary);

root->mcfg_addr = acpi_pci_root_get_mcfg_addr(handle);

acpi_hid = acpi_device_hid(root->device);

if (strcmp(acpi_hid, "PNP0A08") == 0)

root->bridge_type = ACPI_BRIDGE_TYPE_PCIE;

else if (strcmp(acpi_hid, "ACPI0016") == 0)

root->bridge_type = ACPI_BRIDGE_TYPE_CXL;

else

dev_dbg(&device->dev, "Assuming non-PCIe host bridge\n");

negotiate_os_control(root, &no_aspm);

/* Scan the Root Bridge

* Must do this prior to any attempt to bind the root device, as the

* PCI namespace does not get created until this call is made (and

* thus the root bridge's pci_dev does not exist). */

root->bus = pci_acpi_scan_root(root);

if (no_aspm) pcie_no_aspm();

pci_acpi_add_bus_pm_notifier(device);

device_set_wakeup_capable(root->bus->bridge, device->wakeup.flags.valid);

if (hotadd) {

pcibios_resource_survey_bus(root->bus);

pci_assign_unassigned_root_bus_resources(root->bus);

/*This is only called for the hotadd case. For the boot-time

* case, we need to wait until after PCI initialization in

* order to deal with IOAPICs mapped in on a PCI BAR.

* This is currently x86-specific, because acpi_ioapic_add()

* is an empty function without CONFIG_ACPI_HOTPLUG_IOAPIC.

* And CONFIG_ACPI_HOTPLUG_IOAPIC depends on CONFIG_X86_IO_APIC

* (see drivers/acpi/Kconfig).*/

acpi_ioapic_add(root->device->handle);

}

pci_lock_rescan_remove();

pci_bus_add_devices(root->bus);

pci_unlock_rescan_remove();

return 1;

...

}

3.1 negotiate_os_control()

3.1.1 acpi_pci_osc_control_set()

3.2 pci_acpi_scan_root()

1. pci_acpi_setup_ecam_mapping(root): 获取该 RC 的 ECAM space

2. 获取ACPI 上报的该RC的资源， Query _OSC各种 capability

3. 创建 host bridge, 每个RC一个。

4. 创建Root bus

5. 递归扫描下游总线：

a) 桥设备的话，后续创建child bus, 递归扫描

6. 最终从Root bus 向下游递归分配资源

/* Interface called from ACPI code to setup PCI host controller */

struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)

{

struct acpi_pci_generic_root_info *ri;

struct pci_bus *bus, *child;

struct acpi_pci_root_ops *root_ops;

struct pci_host_bridge *host;

ri = kzalloc(sizeof(*ri), GFP_KERNEL);

if (!ri) return NULL;

root_ops = kzalloc(sizeof(*root_ops), GFP_KERNEL);

ri->cfg = pci_acpi_setup_ecam_mapping(root);

root_ops->release_info = pci_acpi_generic_release_info;

root_ops->prepare_resources = pci_acpi_root_prepare_resources;

root_ops->pci_ops = (struct pci_ops *)&ri->cfg->ops->pci_ops;

bus = acpi_pci_root_create(root, root_ops, &ri->common, ri->cfg);

if (!bus) return NULL;

/* If we must preserve the resource configuration, claim now */

host = pci_find_host_bridge(bus);

if (host->preserve_config)

pci_bus_claim_resources(bus);

* Assign whatever was left unassigned. If we didn't claim above, * this will reassign everything.

pci_assign_unassigned_root_bus_resources(bus);

list_for_each_entry(child, &bus->children, node)

pcie_bus_configure_settings(child);

return bus;

}

3.2.1 Definition

3.2.1.1 struct pci_host_bridge {}

struct pci_host_bridge {

struct device dev;

struct pci_bus *bus; /* Root bus */

struct pci_ops *ops;

struct pci_ops *child_ops;

void *sysdata;

int busnr;

int domain_nr;

struct list_head windows; /* resource_entry */

struct list_head dma_ranges; /* dma ranges resource list */

u8 (*swizzle_irq)(struct pci_dev *, u8 *); /* Platform IRQ swizzler */

int (*map_irq)(const struct pci_dev *, u8, u8);

void (*release_fn)(struct pci_host_bridge *);

void *release_data;

unsigned int ignore_reset_delay:1; /* For entire hierarchy */

unsigned int no_ext_tags:1; /* No Extended Tags */

unsigned int no_inc_mrrs:1; /* No Increase MRRS */

unsigned int native_aer:1; /* OS may use PCIe AER */

unsigned int native_pcie_hotplug:1; /* OS may use PCIe hotplug */

unsigned int native_shpc_hotplug:1; /* OS may use SHPC hotplug */

unsigned int native_pme:1; /* OS may use PCIe PME */

unsigned int native_ltr:1; /* OS may use PCIe LTR */

unsigned int native_dpc:1; /* OS may use PCIe DPC */

unsigned int native_cxl_error:1; /* OS may use CXL RAS/Events */

unsigned int preserve_config:1; /* Preserve FW resource setup */

unsigned int size_windows:1; /* Enable root bus sizing */

unsigned int msi_domain:1; /* Bridge wants MSI domain */

/* Resource alignment requirements */

resource_size_t (*align_resource)(struct pci_dev *dev,

const struct resource *res,

resource_size_t start,

resource_size_t size,

resource_size_t align);

unsigned long private[] ____cacheline_aligned;

};

3.2.1.2 struct pci_dev {}

/* The pci_dev structure describes PCI devices */

struct pci_dev {

struct list_head bus_list; /* Node in per-bus list */

struct pci_bus *bus; /* Bus this device is on */

struct pci_bus *subordinate; /* Bus this device bridges to */

void *sysdata; /* Hook for sys-specific extension */

struct proc_dir_entry *procent; /* Device entry in /proc/bus/pci */

struct pci_slot *slot; /* Physical slot this device is in */

unsigned int devfn; /* Encoded device & function index */

unsigned short vendor;

unsigned short device;

…

struct device dev; /* Generic device interface */

…

}

3.2.1.3 struct pci_bus {}

struct pci_bus {

struct list_head node; /* Node in list of buses */

struct pci_bus *parent; /* Parent bus this bridge is on */

struct list_head children; /* List of child buses */

struct list_head devices; /* List of devices on this bus */

struct pci_dev *self; /* Bridge device as seen by parent */

struct list_head slots; /* List of slots on this bus;

protected by pci_slot_mutex */

struct resource *resource[PCI_BRIDGE_RESOURCE_NUM];

struct list_head resources; /* Address space routed to this bus */

struct resource busn_res; /* Bus numbers routed to this bus */

struct pci_ops *ops; /* Configuration access functions */

struct msi_controller *msi; /* MSI controller */

void *sysdata; /* Hook for sys-specific extension */

struct proc_dir_entry *procdir; /* Directory entry in /proc/bus/pci */

unsigned char number; /* Bus number */

unsigned char primary; /* Number of primary bridge */

unsigned char max_bus_speed; /* enum pci_bus_speed */

unsigned char cur_bus_speed; /* enum pci_bus_speed */

#ifdef CONFIG_PCI_DOMAINS_GENERIC

int domain_nr;

#endif

char name[48];

unsigned short bridge_ctl; /* Manage NO_ISA/FBB/et al behaviors */

pci_bus_flags_t bus_flags; /* Inherited by child buses */

struct device *bridge;

struct device dev;

struct bin_attribute *legacy_io; /* Legacy I/O for this bus */

struct bin_attribute *legacy_mem; /* Legacy mem */

unsigned int is_added:1;

};

3.3 pci_acpi_scan_root() -> acpi_pci_root_create()

1. 获取ACPI PCI RC device 上报的资源（PNP0A08:0?/_CRS etc）

2. 扫描该RC 下游bus

struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root,

struct acpi_pci_root_ops *ops,

struct acpi_pci_root_info *info, void *sysdata)

{

int ret, busnum = root->secondary.start;

struct acpi_device *device = root->device;

int node = acpi_get_node(device->handle);

struct pci_bus *bus;

struct pci_host_bridge *host_bridge;

union acpi_object *obj;

info->root = root;

info->bridge = device;

info->ops = ops;

INIT_LIST_HEAD(&info->resources);

snprintf(info->name, sizeof(info->name), "PCI Bus %04x:%02x", root->segment, busnum);

if (ops->init_info && ops->init_info(info))

goto out_release_info;

if (ops->prepare_resources)

ret = ops->prepare_resources(info);

else

ret = acpi_pci_probe_root_resources(info);

if (ret < 0) goto out_release_info;

pci_acpi_root_add_resources(info);

pci_add_resource(&info->resources, &root->secondary);

bus = pci_create_root_bus(NULL, busnum, ops->pci_ops, sysdata, &info->resources);

if (!bus)

goto out_release_info;

host_bridge = to_pci_host_bridge(bus->bridge);

if (!(root->osc_control_set & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL))

host_bridge->native_pcie_hotplug = 0;

if (!(root->osc_control_set & OSC_PCI_SHPC_NATIVE_HP_CONTROL))

host_bridge->native_shpc_hotplug = 0;

if (!(root->osc_control_set & OSC_PCI_EXPRESS_AER_CONTROL))

host_bridge->native_aer = 0;

if (!(root->osc_control_set & OSC_PCI_EXPRESS_PME_CONTROL))

host_bridge->native_pme = 0;

if (!(root->osc_control_set & OSC_PCI_EXPRESS_LTR_CONTROL))

host_bridge->native_ltr = 0;

if (!(root->osc_control_set & OSC_PCI_EXPRESS_DPC_CONTROL))

host_bridge->native_dpc = 0;

if (!(root->osc_ext_control_set & OSC_CXL_ERROR_REPORTING_CONTROL))

host_bridge->native_cxl_error = 0;

* Evaluate the "PCI Boot Configuration" _DSM Function. If it

* exists and returns 0, we must preserve any PCI resource

* assignments made by firmware for this host bridge.

obj = acpi_evaluate_dsm(ACPI_HANDLE(bus->bridge), &pci_acpi_dsm_guid, 1,

DSM_PCI_PRESERVE_BOOT_CONFIG, NULL);

if (obj && obj->type == ACPI_TYPE_INTEGER && obj->integer.value == 0)

host_bridge->preserve_config = 1;

ACPI_FREE(obj);

acpi_dev_power_up_children_with_adr(device);

pci_scan_child_bus(bus);

pci_set_host_bridge_release(host_bridge, acpi_pci_root_release_info, info);

if (node != NUMA_NO_NODE)

dev_printk(KERN_DEBUG, &bus->dev, "on NUMA node %d\n", node);

return bus;

out_release_info:

__acpi_pci_root_release_info(info);

return NULL;

}

3.3.1 pci_acpi_root_prepare_resources()

acpi_pci_root_add()

pci_acpi_scan_root()

root_ops->release_info = pci_acpi_generic_release_info;

root_ops->prepare_resources = pci_acpi_root_prepare_resources;

acpi_pci_root_create()

|-->ops->prepare_resources(info);

| |-->pci_acpi_root_prepare_resources; //Get resource from ACPI(_CRS..)

//获取ACPI PCI RC device 上报的资源（_CRS）

pci_acpi_root_prepare_resources()

acpi_pci_probe_root_resources()

acpi_dev_get_resources（）

__acpi_dev_get_resources(adev, list, .., METHOD_NAME__CRS);

static int pci_acpi_root_prepare_resources(struct acpi_pci_root_info *ci)

{

struct resource_entry *entry, *tmp;

int status;

status = acpi_pci_probe_root_resources(ci);

resource_list_for_each_entry_safe(entry, tmp, &ci->resources) {

if (!(entry->res->flags & IORESOURCE_WINDOW))

resource_list_destroy_entry(entry);

}

return status;

}

3.3.1.1 pci_acpi_root_add_resources();

struct resource ioport_resource = {

.name = "PCI IO",

.start = 0,

.end = IO_SPACE_LIMIT,

.flags = IORESOURCE_IO,

};

EXPORT_SYMBOL(ioport_resource);

struct resource iomem_resource = {

.name = "PCI mem",

.start = 0,

.end = -1,

.flags = IORESOURCE_MEM,

};

EXPORT_SYMBOL(iomem_resource);

static void pci_acpi_root_add_resources(struct acpi_pci_root_info *info)

{

struct resource_entry *entry, *tmp;

struct resource *res, *conflict, *root = NULL;

resource_list_for_each_entry_safe(entry, tmp, &info->resources) {

res = entry->res;

if (res->flags & IORESOURCE_MEM)

root = &iomem_resource;

else if (res->flags & IORESOURCE_IO)

root = &ioport_resource;

else

continue;

* Some legacy x86 host bridge drivers use iomem_resource and

* ioport_resource as default resource pool, skip it. */

if (res == root) continue;

conflict = insert_resource_conflict(root, res);

if (conflict) {

dev_info(&info->bridge->dev,

"ignoring host bridge window %pR (conflicts with %s %pR)\n",

res, conflict->name, conflict);

resource_list_destroy_entry(entry);

}

3.3.2 pci_create_root_bus()

host bridge res 示意图：

Create root bus & host bridge 图一

Create root bus & host bridge 图二

3.3.3 pci_scan_child_bus() -> .. -> pci_scan_single_device()

Root bus 扫描结束：

创建（递归)child bus(第二轮）：

从Root bus 下游bridge 再次扫描设备：

3.4 pci_assign_unassigned_root_bus_resources(bus);

重点是下面

1. __pci_bus_size_bridges()

2. __pci_bus_assign_resources()

具体参考：

Linux Topics (4.1) - PCI Bus resource sizing and assignment (TBD)-CSDN博客

4. Important Routines

4.1 pci_register_host_bridge()

说明：

1. 添加host bridge device: device_add(&bridge->dev);

比如: # /sys/devices/pci0007:00/

# ls /sys/devices/pci0007\:00/
0007:00:00.0 0007:00:01.0 0007:00:03.0 0007:00:05.0 0007:00:07.0

* 这里注册的host bridge device 是根，它的下游才是PCI 总线，上游是ACPI 总线。

所以Host bridge device 没有挂在 /sys/bus/pci 总线上，但是挂在 /sys/device/上面。

2. 注册 host bridge bus device: device_register(&bus->dev);

a) bus->bridge = get_device(&bridge->dev);

b) bus->dev.class = &pcibus_class;

c) bus->dev.parent = bus->bridge;

就是注册了一个设备class 为pcibus_class, parent是添加的 bridge->dev.

比如：

# ls /sys/class/pci_bus/0007:00/ -l

device -> ../../../pci0007:00

#ls /sys/class/pci_bus/0007\:00/device/
0007:00:00.0 0007:00:01.0 0007:00:03.0 0007:00:05.0 0007:00:07.0

static int pci_register_host_bridge(struct pci_host_bridge *bridge)

{

struct device *parent = bridge->dev.parent;

struct resource_entry *window, *next, *n;

struct pci_bus *bus, *b;

resource_size_t offset, next_offset;

LIST_HEAD(resources);

struct resource *res, *next_res;

char addr[64], *fmt;

const char *name;

int err;

bus = pci_alloc_bus(NULL);

bridge->bus = bus;

bus->sysdata = bridge->sysdata;

bus->ops = bridge->ops;

bus->number = bus->busn_res.start = bridge->busnr;

#ifdef CONFIG_PCI_DOMAINS_GENERIC

if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)

bus->domain_nr = pci_bus_find_domain_nr(bus, parent);

else

bus->domain_nr = bridge->domain_nr;

if (bus->domain_nr < 0) {

err = bus->domain_nr;

goto free;

}

#endif

b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);

if (b) {

/* Ignore it if we already got here via a different bridge */

dev_dbg(&b->dev, "bus already known\n");

err = -EEXIST;

goto free;

}

dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus), bridge->busnr);

err = pcibios_root_bridge_prepare(bridge);

/* Temporarily move resources off the list */

list_splice_init(&bridge->windows, &resources);

err = device_add(&bridge->dev); //TBD

bus->bridge = get_device(&bridge->dev);

device_enable_async_suspend(bus->bridge);

pci_set_bus_of_node(bus);

pci_set_bus_msi_domain(bus);

if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) &&

!pci_host_of_has_msi_map(parent))

bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;

if (!parent)

set_dev_node(bus->bridge, pcibus_to_node(bus));

bus->dev.class = &pcibus_class;

bus->dev.parent = bus->bridge;

dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);

name = dev_name(&bus->dev);

err = device_register(&bus->dev); //TBD

if (err) goto unregister;

pcibios_add_bus(bus);

if (bus->ops->add_bus) {

err = bus->ops->add_bus(bus);

if (WARN_ON(err < 0))

dev_err(&bus->dev, "failed to add bus: %d\n", err);

}

/* Create legacy_io and legacy_mem files for this bus */

pci_create_legacy_files(bus);

if (parent) dev_info(parent, "PCI host bridge to bus %s\n", name);

else pr_info("PCI host bridge to bus %s\n", name);

if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)

dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");

/* Coalesce contiguous windows */

resource_list_for_each_entry_safe(window, n, &resources) {

if (list_is_last(&window->node, &resources))

break;

next = list_next_entry(window, node);

offset = window->offset;

res = window->res;

next_offset = next->offset;

next_res = next->res;

if (res->flags != next_res->flags || offset != next_offset)

continue;

if (res->end + 1 == next_res->start) {

next_res->start = res->start;

res->flags = res->start = res->end = 0;

}

/* Add initial resources to the bus */

resource_list_for_each_entry_safe(window, n, &resources) {

offset = window->offset;

res = window->res;

if (!res->flags && !res->start && !res->end)

continue;

list_move_tail(&window->node, &bridge->windows);

if (res->flags & IORESOURCE_BUS)

pci_bus_insert_busn_res(bus, bus->number, res->end);

else

pci_bus_add_resource(bus, res, 0);

if (offset) {

if (resource_type(res) == IORESOURCE_IO)

fmt = " (bus address [%#06llx-%#06llx])";

else

fmt = " (bus address [%#010llx-%#010llx])";

snprintf(addr, sizeof(addr), fmt, (unsigned long long)(res->start - offset),

(unsigned long long)(res->end - offset));

} else

addr[0] = '\0';

dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);

}

down_write(&pci_bus_sem);

list_add_tail(&bus->node, &pci_root_buses);

up_write(&pci_bus_sem);

return 0;

unregister:

put_device(&bridge->dev);

device_del(&bridge->dev);

free:

...

}

4.2 pci_scan_single_device()

4.2.1 pci_scan_device();

4.2.2.1 pci_setup_device(struct pci_dev *dev)

/**
* pci_setup_device - Fill in class and map information of a device
* @dev: the device structure to fill
*
* Initialize the device structure with information about the device's
* vendor,class,memory and IO-space addresses, IRQ lines etc.
* Called at initialisation of the PCI subsystem and by CardBus services.
* Returns 0 on success and negative if unknown type of device (not normal,* bridge or CardBus).
*/
int pci_setup_device(struct pci_dev *dev)
{
u32 class;
u16 cmd;
u8 hdr_type;
int err, pos = 0;
struct pci_bus_region region;
struct resource *res;

hdr_type = pci_hdr_type(dev);

dev->sysdata = dev->bus->sysdata;
dev->dev.parent = dev->bus->bridge;
dev->dev.bus = &pci_bus_type;
dev->hdr_type = hdr_type & 0x7f;
dev->multifunction = !!(hdr_type & 0x80);
dev->error_state = pci_channel_io_normal;
set_pcie_port_type(dev);