################################################################################
HOST 安装必要包
################################################################################
yum -y install fuse fuse-devel
yum -y install kernel-modules-extra
vim /etc/grub2.cfg
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 console=tty0 console=ttyS0,115200n8 iommu=pt intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=6
########################################################
HOST 编译QEMU
################################################
#git clone https://git.qemu.org/git/qemu.git
#cd /home/liufeng/workspace/src/ovs-dpdk/qemu
#git checkout v3.0.0-rc4 -b v3.0.0-rc4
#./configure --prefix=/home/liufeng/workspace/tools/ --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl --disable-git-update
#scripts/git-submodule.sh update ui/keycodemapdb dtc
#make
#sudo make install
################################################################################
HOST DPDK 配置
################################################################################
su
grep HugePages_ /proc/meminfo
modprobe vfio
modprobe vfio-pci
modprobe uio_pci_generic
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*
modprobe vhost
modprobe cuse
dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down
sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3
dpdk-devbind --status
################################################################################
HOST 启动 vhost-switch
################################################################################
编译 example vhost
cd /home/liufeng/workspace/src/ovs-dpdk/dpdk/examples/vhost
#make RTE_SDK=pwd
/…/… RTE_TARGET=x86_64-native-linuxapp-gcc
#make RTE_SDK=pwd
/…/… RTE_TARGET=x86_64-native-linuxapp-gcc V=1
#export RTE_TARGET=x86_64-native-linuxapp-gcc
#export RTE_SDK=pwd
/…/…
rm -rf ./build
#make
#make V=1
make RTE_SDK=pwd
/…/… RTE_TARGET=x86_64-native-linuxapp-gcc
################################################################################
HOST 启动 vhost-switch
################################################################################
cd /home/liufeng/workspace/src/ovs-dpdk/dpdk/examples/vhost
#client mode
./build/vhost-switch -l 0-3 -n 4 --socket-mem 1024 – -p 0x1 --socket-file /tmp/sock0 --client
./build/vhost-switch -l 0-3 -n 4 --socket-mem 1024 – -p 0x1 --socket-file /tmp/sock0 --client –stats 5
#server mode
./build/vhost-switch -l 0-3 -n 4 --socket-mem 1024 – -p 0x1 --socket-file /tmp/sock0
################################################################################
HOST VM 网络配置
################################################################################
systemctl stop abrtd.service
service NetworkManager stop
tunctl -t tap30
brctl addif br1 tap30
brctl show
ifconfig tap30 promisc up
tunctl -t tap31
brctl addif br2 tap31
brctl show
ifconfig tap31 promisc up
brctl addif br2 enp1s0f1
ifconfig enp1s0f1 0
ifconfig br2 192.168.4.1
ping -c 3 192.168.4.2
################################################################################
HOST VM 启动
################################################################################
#qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M
#-object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on
#-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem
#-vnc 0.0.0.0:00 -monitor stdio --enable-kvm
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08
#-netdev tap,id=hostnet1,vhost=on,vhostforce=on,script=no,downscript=no -device virtio-net-pci,netdev=hostnet1,mac=00:00:00:00:00:01
#-netdev tap,id=hostnet2,vhost=on,vhostforce=on,script=no,downscript=no -device virtio-net-pci,netdev=hostnet2,mac=00:00:00:00:00:02
#-chardev socket,id=char1,path=/tmp/sock0,server
#-netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
#qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M
#-object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on
#-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem
#-vnc 0.0.0.0:00 -monitor stdio --enable-kvm
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08
#-chardev socket,id=char1,path=/tmp/sock0,server
#-netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M
-object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device virtio-net-pci,netdev=eth0,mac=12:03:04:05:06:08
-chardev socket,id=char1,path=/tmp/sock0,server
-netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
HMP手动添加设备virtio-net-vhost-user
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M
-object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device virtio-net-pci,netdev=eth0,mac=12:03:04:05:06:08
-chardev socket,id=char1,path=/tmp/sock0,server
-netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce
#hmp 动态添加设备
device_add virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
################################################################################
QEMU DEBUG
################################################################################
gdb /home/liufeng/workspace/tools/bin/qemu-system-x86_64
virtio
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device virtio-net-pci,netdev=eth0,mac=12:03:04:05:06:08 -chardev socket,id=char1,path=/tmp/sock0,server -netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
e1000
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -chardev socket,id=char1,path=/tmp/sock0,server -netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
/vhost-user, server/
gdb --args /home/liufeng/workspace/tools/bin/qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -chardev socket,id=char1,path=/tmp/sock0,server -netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
/vhost-user, client/
gdb --args /home/liufeng/workspace/tools/bin/qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -chardev socket,id=char1,path=/tmp/sock0 -netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
gdb /home/liufeng/workspace/tools/bin/qemu-system-x86_64
e1000
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08
e1000 + virtio
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no -device virtio-net-pci,netdev=eth31,mac=00:00:00:00:00:31
gdb --args /home/liufeng/workspace/tools/bin/qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no -device virtio-net-pci,netdev=eth31,mac=00:00:00:00:00:31,mq=on
b virtio_ioport_write
b virtio_ioport_read
b virtio_net_device_realize
b virtio_net_handle_mq
b virtio_pci_reset
b virtio_pci_dc_realize
e1000 + vhost-user
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -chardev socket,id=char1,path=/tmp/sock0,server -netdev type=vhost-user,id=mynet3,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet3,id=net1,mac=00:00:00:00:00:03
e1000 手动加入VIRTIO设备
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no
gdb --args /home/liufeng/workspace/tools/bin/qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no
QEMU 操作
device_add virtio-net-pci,netdev=eth31,mac=00:00:00:00:00:31
e1000 手动加入VIRTIO设备
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no -s
set args -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no -S -gdb tcp::1234 -s
/home/liufeng/workspace/tools/bin/qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M -object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem -vnc 0.0.0.0:00 -monitor stdio --enable-kvm -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 -netdev type=tap,id=eth31,ifname=tap31,script=no,downscript=no -S -gdb tcp::1234 -s
#gdb
set architecture i386:x86-64:intel
target remote :1234
#file /home/liufeng/workspace/dbg/linux-3.10.0-1127.el7/vmlinux
b schedule
QEMU 操作
device_add virtio-net-pci,netdev=eth31,mac=00:00:00:00:00:31
#brctl addif br2 tap0
#brctl addif br2 tap0
################################################################################
单独编译替换GUEST OS的VIRTIO模块
################################################################################
单独编译替换
进入virtio代码目录
VIRTIO_PCI
cd /home/liufeng/workspace/src/ovs-dpdk/linux-3.10.0-1127.el7/drivers/virtio
#开始编译
#make -C /lib/modules/uname -r
/build M=pwd
clean
make -C /lib/modules/uname -r
/build M=pwd
modules
#拷贝编译结果出来,并使用
#cp *.ko /home/liufeng/tools/modules/
#cd /home/liufeng/tools/modules/
lsmod | grep virtio
modprobe -r virtio_pci
modprobe -r virtio_net
lsmod | grep virtio
modprobe virtio_ring
lsmod | grep virtio
dmesg -c
insmod ./virtio.ko
insmod ./virtio_pci.ko
dmesg -c
#单独编译替换VIRTIO_INET
进入virtio_inet代码目录
cd /home/liufeng/workspace/src/ovs-dpdk/linux-3.10.0-1127.el7/drivers/net
#开始编译
修改Makefile
只剩下:obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
make -C /lib/modules/uname -r
/build M=pwd
clean
make -C /lib/modules/uname -r
/build M=pwd
modules
#拷贝编译结果出来,并使用
#cp *.ko /home/liufeng/tools/modules/
#cd /home/liufeng/tools/modules/
cd /home/liufeng/workspace/src/ovs-dpdk/linux-3.10.0-1127.el7/drivers/virtio
lsmod | grep virtio
modprobe -r virtio_pci
modprobe -r virtio_net
lsmod | grep virtio
modprobe virtio_ring
lsmod | grep virtio
dmesg -c
insmod ./virtio.ko
insmod ./virtio_pci.ko
dmesg -c
cd /home/liufeng/workspace/src/ovs-dpdk/linux-3.10.0-1127.el7/drivers/net
insmod ./virtio_net.ko
lsmod | grep virtio
dmesg -c
################################################################################
VM 内部配置
################################################################################
service NetworkManager stop
ifconfig eth0 192.168.4.5 up
ethtool -i eth0
ping 192.168.4.2
udevadm monitor --udev --kernel --property
#auditctl -w /lib/modules/3.10.0-feng/kernel/drivers/net/virtio_net.ko -p rwxa
auditctl -w /lib/modules/3.10.0-feng/kernel/drivers/virtio/virtio_pci.ko -p rwxa
tailf /var/log/audit/audit.log
################################################################################
VM 内部配置 Test PMD
################################################################################
su
grep HugePages_ /proc/meminfo
sudo modprobe vfio
sudo modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*
modprobe uio_pci_generic
dpdk-devbind --status
sudo ifconfig eth0 down
#sudo ifconfig eth1 down
#sudo ifconfig eth2 down
dpdk-devbind --bind=uio_pci_generic eth0
#dpdk-devbind --bind=uio_pci_generic eth1
#dpdk-devbind --bind=uio_pci_generic eth2
dpdk-devbind --status
动态库情况
#testpmd -l 0-1 -d /home/liufeng/workspace/src/dpdk/sdk/lib/librte_pmd_virtio.so – -i
testpmd -l 0-1 – -i
testpmd -l 0-1 --socket-mem 128 – --burst=64 -i
show port stats all
start tx_first
show port stats all
udevadm monitor --udev --kernel --property
#auditctl -w /lib/modules/3.10.0-feng/kernel/drivers/net/virtio_net.ko -p rwxa
auditctl -w /lib/modules/3.10.0-feng/kernel/drivers/virtio/virtio_pci.ko -p rwxa
tailf /var/log/audit/audit.log
###################################
格式化文件
###################################
set invlist
%s/\s+$//g
set ts=4
set expandtab
%retab!