BA-OVS-DPDK-VM.txt

################################################################################
# OVS DPDK VM 安装
################################################################################ 

#vim ~/.bash_profile
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#DPDK_SDK=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#export DPDK_SDK
#
#LD_LIBRARY_PATH=$DPDK_SDK/lib:$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH
#export LD_LIBRARY_PATH
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#source ~/.bash_profile
#
#
#sudo vim /etc/grub2.cfg
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=6
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#
#
#/home/liufeng/workspace/src/ovs-dpdk
#git clone git@github.com:DPDK/dpdk.git
#git clone git@github.com:openvswitch/ovs.git
#
#
#cd /home/liufeng/workspace/src/ovs-dpdk/dpdk
#git checkout v19.11 -b v19.11
#vim config/common_base
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#CONFIG_RTE_BUILD_SHARED_LIB=y
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

#make config T=x86_64-native-linuxapp-gcc
#sed -ri 's,(PMD_PCAP=).*,\1y,' build/.config
#mkdir /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
##sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr
#sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/
#
#vim /etc/ld.so.conf.d/dpdk.conf
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#ldconfig
#
#sudo yum install autoconf.noarch automake.noarch libtool.x86_64 python3-pip.noarch

#cd /home/liufeng/workspace/src/open/ovs
#git checkout v2.13.0 -b v2.13.0
#./boot.sh
#./configure \
#--with-dpdk=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk \
#--prefix=/home/liufeng/workspace/tools/ \
#--exec-prefix=/home/liufeng/workspace/tools/ \
#--sysconfdir=/home/liufeng/workspace/tools/etc \
#--localstatedir=/home/liufeng/workspace/tools/var

#make
#sudo make install

su 

sudo modprobe vfio
sudo modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*

sudo modprobe uio_pci_generic
#sudo insmod /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko

dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down

sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3

dpdk-devbind --status

mount -t hugetlbfs -o pagesize=1G none /dev/hugepages
grep HugePages_ /proc/meminfo

#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/

#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db

#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema

# 启动ovsdb server
ovsdb-server \
/home/liufeng/workspace/tools/etc/openvswitch/conf.db \
-vconsole:emer -vsyslog:err -vfile:info \
--remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
--remote=db:Open_vSwitch,Open_vSwitch,manager_options \
--private-key=db:Open_vSwitch,SSL,private_key \
--certificate=db:Open_vSwitch,SSL,certificate \
--bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \
--detach --monitor

ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true

#  初始化dpdk
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait init
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 指定的sockets从hugepages预先分配的内存
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0"
# 指定在某些core上运行
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x02

#查看自定义的dpdk参数
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-socket-mem
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:pmd-cpu-mask
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-init

#启动vswitchd进程
ovs-vswitchd \
unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
-vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \
--detach --monitor

top -H
ps -eLo pid,psr,comm | grep pmd

ovs-vsctl --version
ovs-vsctl show
#把dpdk端口加入网桥
#ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
#ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0
#ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1 
#ovs-vsctl add-port br0 vhost-user-1 -- set Interface vhost-user-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0"
#ovs-vsctl add-port br0 vhost-user-1-1 -- set Interface vhost-user-1-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0-1"
#ovs-vsctl add-port br0 vhost-user-2 -- set Interface vhost-user-2 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock2" 

ovs-ofctl dump-flows br0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0

ovsdb-client dump

#git clone https://git.qemu.org/git/qemu.git
#/home/liufeng/workspace/src/ovs-dpdk/qemu
#git checkout v3.0.0-rc4 -b v3.0.0-rc4
#./configure --prefix=/home/liufeng/workspace/tools/ --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-debug-info --disable-sdl --disable-git-update
#scripts/git-submodule.sh update ui/keycodemapdb dtc
#make
#sudo make install

#pushd /home/kvm/disk
#qemu-img create -f qcow2 -o size=50G vm3.img
#popd

#vim /etc/yum.repos.d/nux-misc.repo
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#[nux-misc]
#name=Nux Misc
#baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/
#enabled=0
#gpgcheck=1
#gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#yum --enablerepo=nux-misc install tunctl

#建立VM网络
tunctl -t tap30
tunctl -t tap33
brctl addif br1 tap30
brctl addif br1 tap33
brctl show
ifconfig tap30 promisc up
ifconfig tap33 promisc up

#安装：
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 -boot d \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38

# 普通启动 VM3
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38

# 查看OVS状态
tailf /home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log

# vhost user 启动 VM0 
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 \
-chardev socket,id=char1,path=/tmp/sock0,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:01

#进入VM0
service NetworkManager stop
ifconfig eth0 192.168.5.1/24 up
tcpdump -i eth0

#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 （可以ping通）
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 （可以ping通）

# vhost user 启动 VM3 
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm3.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:30 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap33,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38 \
-chardev socket,id=char1,path=/tmp/sock2,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:03

# 进入VM3
# service NetworkManager stop
ifconfig eth0 192.168.5.2/24 up
ping 192.168.5.1 

#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 （可以ping通）
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 （可以ping通）

#删除VM网络
tunctl -d tap30

# 解开内核SRC RPM包
rpm2cpio kernel-3.10.0-1127.el7.src.rpm | cpio -idmv