一. Host OS中QEMU中虚拟的EDU PCI设备
下面这里,会注册一个 TYPE_PCI_EDU_DEVICE,而它继承于 TYPE_PCI_DEVICE,代码如下:
从下面这里可以看出,该EDU设备的实例化接口是:pci_edu_realize()
同时,也可以看出它的 venoder_id的device_id,这2个参数是关键:
#define PCI_VENDOR_ID_QEMU 0x1234
所以,EDU这个PCI虚拟的PCI设备,它的 VENDOER ID 和 DEVICE ID 分别如下:
Vendor ID: 0x1234
Device ID: 0x11e8
知道一个PCI设备的这2个ID是非常关键的,只要我们知道了 Vendor ID和Device ID,那么,我们就可以以 Guest OS中,根据这2个ID来匹配 PCI设备驱动了,可以说,如果不关心PCI总线的细节,只关心PCI设备驱动的话,那么Vendor ID的Device ID是联系PCI设备与PCI驱动的纽带。
如下是pci_edu_realize(),可以看到,它注册了 PCI设备的BAR空间,该空间的操作函数是:edu_mmio_ops
当 Guest OS对EDU虚拟设备进行读写的时候,会被Host捕捉到,从而调用下面的 edu_mmio_read()和edu_mmio_write()函数中来,整体流程还是比较好理解的,至于为什么Host能捕捉到Guest的读写操作,这里由于Guest的每一条Load/Store指令,都是由Host代为执行的,那么,Host当然就都可以感知到这条指令,检测到了 Load/Store指令,再根据要读/写的地址,就能查找到该地址所在的MemoryRegion 对应的MemoryRegionOps,从而调用到MemoryRegionOps->read()/write()接口。
二. Guest Linux中的 PCIE EDU Device 驱动
$ cat my_pcie_edu_driver.c
#include <linux/bitops.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/pm_runtime.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/printk.h>
#include <linux/workqueue.h>
#define _NAME "my_pcie_edu"
#define EDU_VENDOR 0x1234
#define EDU_DEVICE 0x11e8
struct my_pcie_edu_hwreg {
struct device *dev;
struct resource *res;
void *registers;
};
int my_pcie_edu_bar_dump(struct device *dev, struct my_pcie_ep_hwreg *hwreg, struct resource *res, int idx)
{
char const *name = dev_name(dev);
unsigned int reg_val;
int ret;
int i;
hwreg->dev = dev;
hwreg->res = request_mem_region(res->start, resource_size(res), name);
if (hwreg->res == NULL) {
printk("Failed to request mem region. start=0x%llx, size=0x%llx.",
res->start, resource_size(res));
return -EINVAL;
}
hwreg->registers = ioremap(res->start, resource_size(res));
if (hwreg->registers == NULL) {
printk("Failed to iomap region. start=0x%llx, size=0x%llx.",
res->start, resource_size(res));
ret = -ENOMEM;
goto release_mem;
}
//test write bar space
for( i = 0; i < 0x20; i += sizeof(unsigned int))
{
*(volatile unsigned int *)(hwreg->registers + i) = 0xABABCD00 + i;
}
//test read bar spcae
for( i = 0; i < 0x20; i += sizeof(unsigned int))
{
reg_val = *(volatile unsigned int *)(hwreg->registers + i);
}
return 0;
unmap_io:
iounmap(hwreg->registers);
release_mem:
release_mem_region(res->start, resource_size(res));
return ret;
}
static int my_pcie_edu_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
{
static unsigned int dev_id;
struct resource irqres = {
.start = pdev->irq,
.end = pdev->irq,
.flags = 0
};
pdev->dev.id = dev_id++;
my_pcie_edu_bar_dump(&pdev->dev, &hwreg, &pdev->resource[0], 0);
}
static void my_pcie_edu_remove(struct pci_dev *pdev)
{
printk("My pcie edu driver remove\r\n");
}
static struct pci_device_id my_pcie_edu_device_id[] = {
{ PCI_DEVICE(EDU_VENDOR, EDU_DEVICE) },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, my_pcie_edu_device_id);
static struct pci_driver my_pcie_edu_driver = {
.name = _NAME,
.id_table = my_pcie_edu_device_id,
.probe = my_pcie_edu_probe,
.remove = my_pcie_edu_remove
};
static int __exit my_pcie_edu_init(void)
{
printk("My pcie edu driver init\r\n");
int ret;
ret = pci_register_driver(&my_pcie_edu_driver);
if (ret != 0) {
printk("Failed to register my_pcie_edu_driver.\n");
return -1;
}
return 0;
}
static void __exit my_pcie_edu_exit(void)
{
printk("My pcie edu driver exit\r\n");
}
module_init(my_pcie_edu_init);
module_exit(my_pcie_edu_exit);
MODULE_LICENSE("GPL");
写一个Makefile,将上述驱动编译成 ko 模块文件:
$ cat Makefile
obj-m +=my_pcie_edu_driver.o
KDIR:=/xxx/linux-5.4 //update to yourself real directionary
all:
@make -C $(KDIR) M=$(PWD) modules
@rm -fr .tmp_versions *.o *.mod.o *.mod.c *.bak *.symvers *.markers *.unsigned *.order *~ .*.*.cmd .*.*.*.cmd
clean:
@make -C $(KDIR) M=$(PWD) modules clean
编译上述驱动得到 my_pcie_edu_driver.ko
三. 在Host中启动QEMU并加载edu虚拟设备
qemu-system-aarch64 -M virt -cpu cortex-a57 -smp 4 -m 2G -kernel ./linux-guest/linux-5.19/build/arch/arm64/boot/Image -initrd ramfs/initramfs.cpio.gz -nographic -append “init=/init console=ttyAMA0” -device edu
启动qemu时,加上 -device edu 这个参数后,edu 设备将会被正常地初始化,等虚拟的linux启动成功后,通过 lspci 命令就可以看到 edu 这个pci 设备已经被正确地识别到了:
00.02.0 Class 00ff: 1234:11e8
四. 在 Guest Linux中加载my_pcie_edu_driver.ko
可以通过NFS的方式,将 my_pcie_edu_driver.ko 拷贝到Guest Linux的ramfs中,具体步骤可以自行摸索,这里不再详述。
当加载后,由于my_pcie_edu_driver.ko中的vendor id和device id将会与1234:11e8 匹配成功,从而正确地调用到my_pcie_edu_probe()函数中来,这里,就可以通过request_mem_region以及ioremap将EDU设备的BAR空间映射过来,后面就可以正常读写EDU设备的BAR空间了。