virt-install支持VF

原始CentOS自带QEMU是不支持RDMA的，需要重新编译，可以参考我的上一篇文档：

QEMU源码编译CentOS

物理机BIOS已开启SR-IOV，就开始使用SR-IOV

SR-IOV创建出的虚拟网卡称为VF（Virtual Function），如下命令可以查看网卡物理端口ens4f0/1（称PF）最大支持创建的VF均为8个

# cat /sys/class/net/ens4f0/device/sriov_totalvfs
8
# cat /sys/class/net/ens4f1/device/sriov_totalvfs
8
ens4f0单个网口虚拟出6个VF

# echo 6 > /sys/class/net/ens4f0/device/sriov_numvfs
 
# lspci|grep Mellanox
b1:00.0 Ethernet controller: Mellanox Technologies MT2894 Family [ConnectX-6 Lx]
b1:00.1 Ethernet controller: Mellanox Technologies MT2894 Family [ConnectX-6 Lx]
b1:00.2 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.3 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.4 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.5 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.6 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
b1:00.7 Ethernet controller: Mellanox Technologies ConnectX Family mlx5Gen Virtual Function
 
# ip link |grep ens4
261: ens4f0v0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
262: ens4f0v1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
263: ens4f0v2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
264: ens4f0v3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
265: ens4f0v4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
266: ens4f0v5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
18: ens4f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
19: ens4f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
 
# ip link show ens4f0v0
261: ens4f0v0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 56:ba:79:b5:fb:3a brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v1
262: ens4f0v1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 42:f9:c8:62:be:fd brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v2
263: ens4f0v2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 2e:2b:21:22:a7:da brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v3
264: ens4f0v3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 22:cd:f8:8e:8b:39 brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v4
265: ens4f0v4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether b6:b1:22:d5:28:46 brd ff:ff:ff:ff:ff:ff
[root@stgExt1 qemu]# ip link show ens4f0v5
266: ens4f0v5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether be:64:4f:36:e0:f7 brd ff:ff:ff:ff:ff:ff

虚拟网卡已经准备就绪了，开始创建KVM了，并且制定mac=${VF MAC}

rm -rf /var/lib/libvirt/images/mytest.raw
qemu-img create -f raw /var/lib/libvirt/images/mytest.raw 120G
chown qemu:qemu /var/lib/libvirt/images/mytest.raw

virt-install \
        --name mytest \
        --vcpus 2 \
        --cpu host-passthrough \
        --memory 8348 \
        --location ./CentOS8.4_x86_64.iso,kernel=images/pxeboot/vmlinuz,initrd=images/pxeboot/initrd.img \
        --extra-args "inst.stage2=hd:LABEL=XXXXXX inst.ks=file:/ks_preload.cfg mgmt_net=192.128.134.112/24@192.128.134.1@preload-mytest@ sys_drive=auto" \
        --disk path=/var/lib/libvirt/images/mytest.img,format=qcow2,size=120,bus=scsi,cache=writeback \
        "--qemu-commandline= -drive file=/var/lib/libvirt/images/mytest.raw,format=raw,if=none,id=D1 -device nvme,drive=D1,serial=1" \
        --os-variant centos7.0 \
        --network network=host-bridge,model=virtio \
        --network network=cluster_E,model=virtio,mac=56:ba:79:b5:fb:3a \
        --graphics vnc \
        --noautoconsole

我还需要使用VF passthrough功能将虚拟网口映射到KVM虚拟机中，上述virt-install命令中，我尝试添加新参数 --hostdev b1:00.4，但是报错如下：

WARNING  ./CentOS8.4-x86_64.iso 可能不能被管理程序访问。您将需要授予 'qemu' 用户搜索以下目录的权限： ['']

开始安装......
搜索文件 vmlinuz......                                                                                                       | 9.6 MB  00:00:00     
搜索文件 initrd.img......                                                                                                    |  72 MB  00:00:00     
正在分配 'mytest.img'                                                                                                 | 120 GB  00:00:00     
ERROR    unsupported configuration: host doesn't support passthrough of host PCI devices
正在删除磁盘 'mytest.img'                                                                                               |    0 B  00:00:00     
域安装失败，您可以运行下列命令重启您的域：
'virsh start virsh --connect qemu:///system start xiaonan_test3'
否则请重新开始安装。

virt-install不支持后，又尝试qemu-system-x86直接映射VF依旧失败 no iommu_group found: No such file or directory

# qemu-system-x86 -name xiaonan_test2 -machine pc -cpu host -m 4G  -drive file=/var/lib/libvirt/images/xiaonan_test2.qcow2,format=qcow2 -device vfio-pci,host=b1:00.4,id=net0,multifunction=on -boot c

qemu-kvm: -device vfio-pci,host=b1:00.4,id=net0,multifunction=on: vfio 0000:b1:00.4: no iommu_group found: No such file or directory

再尝试virsh命令关联PCI，但是遇到 error: argument unsupported: VFIO device assignment is currently not supported on this system

# virsh nodedev-list --tree
# virsh nodedev-detach pci_0000_b1_00_4
setlocale: No such file or directory
error: Failed to detach device pci_0000_b1_00_4
error: argument unsupported: VFIO device assignment is currently not supported on this system