################################################################################
# OVS DPDK VM 安装
################################################################################
#vim ~/.bash_profile
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#DPDK_SDK=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#export DPDK_SDK
#
#LD_LIBRARY_PATH=$DPDK_SDK/lib:$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH
#export LD_LIBRARY_PATH
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#source ~/.bash_profile
#
#
#sudo vim /etc/grub2.cfg
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=6
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#
#
#/home/liufeng/workspace/src/ovs-dpdk
#git clone git@github.com:DPDK/dpdk.git
#git clone git@github.com:openvswitch/ovs.git
#
#
#cd /home/liufeng/workspace/src/ovs-dpdk/dpdk
#git checkout v19.11 -b v19.11
#vim config/common_base
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#CONFIG_RTE_BUILD_SHARED_LIB=y
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#make config T=x86_64-native-linuxapp-gcc
#sed -ri 's,(PMD_PCAP=).*,\1y,' build/.config
#mkdir /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
##sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr
#sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/
#
#vim /etc/ld.so.conf.d/dpdk.conf
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#ldconfig
#
#sudo yum install autoconf.noarch automake.noarch libtool.x86_64 python3-pip.noarch
#cd /home/liufeng/workspace/src/open/ovs
#git checkout v2.13.0 -b v2.13.0
#./boot.sh
#./configure \
#--with-dpdk=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk \
#--prefix=/home/liufeng/workspace/tools/ \
#--exec-prefix=/home/liufeng/workspace/tools/ \
#--sysconfdir=/home/liufeng/workspace/tools/etc \
#--localstatedir=/home/liufeng/workspace/tools/var
#make
#sudo make install
su
sudo modprobe vfio
sudo modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*
sudo modprobe uio_pci_generic
#sudo insmod /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko
dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down
sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3
dpdk-devbind --status
mount -t hugetlbfs -o pagesize=1G none /dev/hugepages
grep HugePages_ /proc/meminfo
#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/
#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db
#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema
# 启动ovsdb server
ovsdb-server \
/home/liufeng/workspace/tools/etc/openvswitch/conf.db \
-vconsole:emer -vsyslog:err -vfile:info \
--remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
--remote=db:Open_vSwitch,Open_vSwitch,manager_options \
--private-key=db:Open_vSwitch,SSL,private_key \
--certificate=db:Open_vSwitch,SSL,certificate \
--bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \
--detach --monitor
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 初始化dpdk
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait init
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 指定的sockets从hugepages预先分配的内存
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0"
# 指定在某些core上运行
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x02
#查看自定义的dpdk参数
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-socket-mem
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:pmd-cpu-mask
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-init
#启动vswitchd进程
ovs-vswitchd \
unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
-vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \
--detach --monitor
top -H
ps -eLo pid,psr,comm | grep pmd
ovs-vsctl --version
ovs-vsctl show
#把dpdk端口加入网桥
#ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
#ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0
#ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1
#ovs-vsctl add-port br0 vhost-user-1 -- set Interface vhost-user-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0"
#ovs-vsctl add-port br0 vhost-user-1-1 -- set Interface vhost-user-1-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0-1"
#ovs-vsctl add-port br0 vhost-user-2 -- set Interface vhost-user-2 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock2"
ovs-ofctl dump-flows br0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0
ovsdb-client dump
#git clone https://git.qemu.org/git/qemu.git
#/home/liufeng/workspace/src/ovs-dpdk/qemu
#git checkout v3.0.0-rc4 -b v3.0.0-rc4
#./configure --prefix=/home/liufeng/workspace/tools/ --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl --disable-git-update
#scripts/git-submodule.sh update ui/keycodemapdb dtc
#make
#sudo make install
#pushd /home/kvm/disk
#qemu-img create -f qcow2 -o size=50G vm3.img
#popd
#vim /etc/yum.repos.d/nux-misc.repo
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#[nux-misc]
#name=Nux Misc
#baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/
#enabled=0
#gpgcheck=1
#gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#yum --enablerepo=nux-misc install tunctl
#建立VM网络
tunctl -t tap30
tunctl -t tap33
brctl addif br1 tap30
brctl addif br1 tap33
brctl show
ifconfig tap30 promisc up
ifconfig tap33 promisc up
#安装:
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 -boot d \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38
# 普通启动 VM3
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38
# 查看OVS状态
tailf /home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log
# vhost user 启动 VM0
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 \
-chardev socket,id=char1,path=/tmp/sock0,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:01
#进入VM0
service NetworkManager stop
ifconfig eth0 192.168.5.1/24 up
tcpdump -i eth0
#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 (可以ping通)
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 (可以ping通)
# vhost user 启动 VM3
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm3.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:30 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap33,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38 \
-chardev socket,id=char1,path=/tmp/sock2,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:03
# 进入VM3
# service NetworkManager stop
ifconfig eth0 192.168.5.2/24 up
ping 192.168.5.1
#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 (可以ping通)
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 (可以ping通)
#删除VM网络
tunctl -d tap30
# 解开内核SRC RPM包
rpm2cpio kernel-3.10.0-1127.el7.src.rpm | cpio -idmv