KVM环境初搭建【工作笔记】

最新推荐文章于 2023-08-26 12:37:53 发布

leoufung

最新推荐文章于 2023-08-26 12:37:53 发布

阅读量1.1k

点赞数

分类专栏：虚拟化工作

本文链接：https://blog.csdn.net/leoufung/article/details/107040519

版权

虚拟化同时被 2 个专栏收录

141 篇文章 32 订阅

订阅专栏

工作

105 篇文章 2 订阅

订阅专栏

#####################################################################################
#安装系统
#####################################################################################
# 按F2进入BIOS
# 在CPU 配置中的socket1和socket0中查看是否激活了Inter vt-d的功能

# 网络配置接口 ens117f3
# 参考该服务器的接口配置 /etc/sysconfig/network-scripts/ifcfg-ens117f3
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
DEVICE=eno1
HWADDR=6c:0b:84:92:e0:32
ONBOOT=yes
BOOTPROTO=none
IPADDR=10.111.95.32
NETMASK=255.255.192.0
GATEWAY=10.111.127.254
IPV6INIT=yes
IPV6_AUTOCONF=no
DHCPV6=no
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

如果添加网卡到网桥br0
ifcfg-br0
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
DEVICE=br0
TYPE=bridge
ONBOOT=yes
BOOTPROTO=static
IPADDR=10.111.95.32
NETMASK=255.255.192.0
GATEWAY=10.111.127.254
DNS1=10.96.1.18
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

ifcfg-eno1
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
TYPE=Ethernet
DEVICE=eno1
NAME=eno1
HWADDR=6c:0b:84:92:e0:32
ONBOOT=yes
BOOTPROTO=none
DEFROUTE=yes
BRIDGE=br0
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

#重启网络服务
systemctl restart network.service
# 添加路由
route add default gw 10.74.120.1
route add -net 10.74.0.0 netmask 255.255.0.0 gw 10.74.120.1

#查看libvirt的版本
virsh version

#解除网桥绑定
virsh iface-unbridge br0

#查看libvirt所管理的接口
virsh iface-list

#将接口添加到网桥
virsh iface-bridge ens117f3 br0
virsh iface-bridge enp0s25 br0

#建立/dev/loop0
mount -o loop win7.iso /mnt

#修改libvirt权限
vim /etc/libvirt/qemu.conf
# 修改如下字段
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
user = "root"
group = "root"
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
systemctl restart libvirtd.service
systemctl status libvirtd.service

#然后再注释掉这两行
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#user = "root"
#group = "root"
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
systemctl restart libvirtd.service
systemctl status libvirtd.service

#修改
#/usr/lib/udev/rules.d/80-kvm.rules 参考40刀片配置

# 关闭SELINUX
setenforce 0
vim /etc/selinux/config
# 修改如下字段
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
SELINUX=disabled
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#查看当前状态
getenforce

#查看KVM模块
lsmod | grep kvm

#建立/dev/loop0
mount -o loop win7.iso /mnt
#系统就会自动建立/dev/loop0
#使用了/dev/loop0后就会自动产生/dev/loop1

#查看已经建立的/dev/loop
losetup -l

#查看下一个建立的/dev/loopn
losetup -f

losetup -o 32256 /dev/loop0 /home/kvm/iso/CgelImg
losetup -l
mount -o loop /dev/loop0 /home/kvm/mnt

######################################################################################
# 关闭防火墙
######################################################################################
service firewalld stop
systemctl disable firewalld.service
systemctl is-enabled firewalld.service

######################################################################################
# Samba配置
######################################################################################
yum install samba.x86_64
rpm -qa | grep samba
smbpasswd -a root
vim /etc/samba/smb.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[root]
path = /home/liufeng
available = yes
browsealbe = yes
public = yes
writable = yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
service smb restart
service nmb restart
systemctl enable smb.service
systemctl enable nmb.service

#修改cgel_lf.xml

#####################################################################################
#挂载IMAGE文件中的内容
#####################################################################################
mkdir mnt
losetup -o 32256 /dev/loop2 iso/CgelImg
mount -o loop /dev/loop2 mnt

#####################################################################################
#通过修改GRUB来选择启动不同的内核和文件系统
#####################################################################################
cd mnt
[root@zxvei-b2 image]# cat grub/grub.conf
# grub.conf generated by anaconda
#
# Note that you do not have to rerun grub after making changes to this file
# NOTICE: You have a /boot partition. This means that
# all kernel and initrd paths are relative to /boot/, eg.
# root (hd0,0)
# kernel /vmlinuz-version ro root=/dev/mapper/VolGroup-lv_root
# initrd /initrd-[generic-]version.img
#boot=/dev/hda
default=0
timeout=5
title CGEL4.x
root (hd0,0)
kernel /bzImage ro root=/dev/ram console=ttyS0,115200
initrd /initrd_wyc.gz

#####################################################################################
#通过 qemu-img 制作虚拟硬盘
#####################################################################################
qemu-img create -f raw cgel.img 2G
qemu-img create -f qcow2 -o size=20G win8-64.img
qemu-img create -f qcow2 -o size=20G win8-32.img
qemu-img create -f qcow2 -o size=20G win81-32.img
qemu-img create -f qcow2 -o size=10G CentOS65-32.img
#qemu-img create -f qcow2 -b 1G.img temp.qcow
#qemu-img create -f qcow2 -o size=10G temp.qcow
mkfs -t ext3 cgel.img
# 修改backing file的位置
qemu-img rebase cirros.disk -b /home/kvm/nfv/cirros/backup.disk

#####################################################################################
#通过 dd 制作虚拟硬盘
#####################################################################################
# 1024 * 1024 K = 1048576 K = 1G
dd if=/dev/zero of=1G.img bs=1k count=1048576
mkfs.ext2 1G.img
#修改cgel_lf.xml
#进入虚拟机后
mkdir /mnt
mount /dev/hdb /mnt

#####################################################################################
#修改文件系统内容
#####################################################################################
mkdir rootfs
cp mnt/initrd_wyc.gz rootfs/
cd rootfs
# 解开文件系统
gunzip -d initrd_wyc.gz
cpio -i < initrd_wyc
rm initrd_wyc

#修改文件系统内容
#..................

#压缩文件系统，并替换老的文件系统
find . | cpio -H newc -o | gzip -9 > ../initrd_cpio_glibc.gz
cd ..
mv initrd_cpio_glibc.gz mnt/

#####################################################################################
# 创建并启动虚拟机
#####################################################################################
virsh create cgel.xml
#查看启动命令和日志
cat /var/log/libvirt/qemu/cgel_lf_kvm.log

# 看虚拟机情况
virsh list

#停止虚拟机
virsh destroy 43

#####################################################################################
#登陆虚拟机
#####################################################################################
telnet 0.0.0.0 5000

#挂载硬盘
mkdir /mnt
mount /dev/hdb /mnt
sync

#查看硬盘情况
mount
fdisk -l

#配置网络
ifconfig eth0 10.74.169.73
route add -net 10.74.0.0 netmask 255.255.0.0 gw 10.74.169.254
route -n

############################################
# VNC管理命令
############################################
vncserver -list
vncserver -kill :2

#####################################################################################
# 手工启动QEMU
#####################################################################################
#/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 -kernel my/bzImage -initrd my/initrd_cpio_glibc.gz
/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 -kernel my/bzImage -initrd my/initrd_cpio_glibc.gz -vnc :2

/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 my/bzImage

#####################################################################################
# 图形界面安装WIN7
#####################################################################################
# 下载ISO ISO 文件所在位置 10.74.169.44 root/root123
# 44 上使用： scp win7.img root@10.74.169.72:/home/liufeng/
# 制作硬盘
qemu-img create -f raw win7.img 20G
# 制作XML文件，启动命令解释见win7.xml

# 设置VNC，realvnc安装好后，避免连接虚拟机一闪而过，依次选择“options”->"advanced"->"expert"->"colourlevel"，将这项改为“full”即可

#启动虚拟机
virsh create xml/win7.xml

# 连接VNC 10.74.169.72:5910

# 后面驱动去掉从CDROM启动，并直接使用win7.img即可

# win7 激活密钥
J783Y-JKQWR-677Q8-KCXTF-BHWGC
C4M9W-WPRDG-QBB3F-VM9K8-KDQ9Y
2VCGQ-BRVJ4-2HGJ2-K36X9-J66JG
MGX79-TPQB9-KQ248-KXR2V-DHRTD
FJHWT-KDGHY-K2384-93CT7-323RC
THHH2-RKK9T-FX6HM-QXT86-MGBCP
KH2J9-PC326-T44D4-39H6V-TVPBY
D8BMB-BVGMF-M9PTV-HWDQW-HPCXX
TFP9Y-VCY3P-VVH3T-8XXCC-MF4YK
236TW-X778T-8MV9F-937GT-QVKBB
87VT2-FY2XW-F7K39-W3T8R-XMFGF
KH2J9-PC326-T44D4-39H6V-TVPBY
TFP9Y-VCY3P-VVH3T-8XXCC-MF4YK
J783Y-JKQWR-677Q8-KCXTF-BHWGC
C4M9W-WPRDG-QBB3F-VM9K8-KDQ9Y
2VCGQ-BRVJ4-2HGJ2-K36X9-J66JG
MGX79-TPQB9-KQ248-KXR2V-DHRTD
FJHWT-KDGHY-K2384-93CT7-323RC

# 网络IP 10.74.169.75 网关 10.74.169.254 DNS 10.41.132.9

#####################################################################################
# 图形界面安装RHEL
#####################################################################################
# 下载ISO ISO 文件所在位置 10.74.169.44 root/root123
# 44 上使用： scp rhel-server-7.0-x86_64-dvd.iso root@10.74.169.72:/home/liufeng/iso
# 制作硬盘
qemu-img create -f raw rhel.img 20G
# 制作XML文件，启动命令解释见rhel_lf.xml

#启动虚拟机
virsh create xml/rhel_lf.xml

# 连接VNC 10.74.169.72:5910

# 后面驱动去掉从CDROM启动，并直接使用rhel_lf.img即可

# 选择GUI 安装，选择必须要的服务；选择

# 网络IP 10.74.169.75 网关 10.74.169.254 DNS 10.41.132.9
cd /etc/sysconfig/network-scripts
vi ifcfg-ens3
>>HWADDR=52:54:00:85:D2:BA
>>TYPE=Ethernet
>>BOOTPROTO=static
>>DEFROUTE=yes
>>PEERDNS=yes
>>PEERROUTES=yes
>>IPV4_FAILURE_FATAL=no
>>IPV6INIT=yes
>>IPV6_AUTOCONF=yes
>>IPV6_DEFROUTE=yes
>>IPV6_PEERDNS=yes
>>IPV6_PEERROUTES=yes
>>IPV6_FAILURE_FATAL=no
>>NAME=ens3
>>UUID=c59f0d06-7774-499a-8c56-47dfe61dbf39
>>ONBOOT=yes
>>IPADDR=10.74.169.76
>>NETMASK=255.255.254.0
>>GATEWAY=10.74.169.254

#################################################3
# 迁移的步骤
###################################################
# NFS操作
#主机1：10.74.169.79
#主机2：10.74.169.72
#在主机1上启动nfs服务器
vim /etc/exprots 添加内容：/home/wyc/CGSL-V4 10.74.169.0/24(rw)
systemctl start nfs-server.service
#主机2上执行操作：
mount 10.74.169.79:/home/wyc/CGSL-V4 /home/wyc/CGSL-V4
mount 10.74.120.168:/home/cgel_img_nfs /home/cgel_img_nfs/
mount 10.74.120.219:/home/iso /home/iso

losetup -o 32256 /dev/loop0 /home/kvm/iso/CgelImg
losetup -l
mount -o loop /dev/loop0 /home/kvm/mnt

#制作登陆key在主机1上执行下列命令，主机2上也可执行（ssh登录不用输入密码）。
ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub root@10.74.120.172
ssh-copy-id -i ~/.ssh/id_rsa.pub root@10.74.120.153

ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub root@10.74.120.171

#设定主机名称
hostnamectl set-hostname lf-127
reboot

#修改uuid，解决error: internal error: Attempt to migrate guest to the same host 00020003-0004-0005-0006-000700080009问题
#修改后需要重启libvirtd
sed -i "/#host_uuid/ahost_uuid = \"`uuidgen`\"" /etc/libvirt/libvirtd.conf
systemctl restart libvirtd.service
systemctl status libvirtd.service

#添加对端主机名称的DNS，解决error: Unable to resolve address 'zxvei-b2' service '49152': Name or service not known
vim /etc/hosts 添加：
10.74.120.172 zxvei-b2
10.74.121.127 lf-127
#开始迁移
virsh migrate cgel_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate cgel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.9/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.8/system --unsafe --verbose

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

172 <==> 171
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose --compressed
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose --compressed

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose tcp://10.74.120.172
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose tcp://10.74.120.171

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.9/system --unsafe --verbose tcp://10.74.120.9
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.8/system --unsafe --verbose tcp://10.74.120.8

virsh migrate win7_kvm_nfs --live qemu+ssh://192.168.0.9/system --unsafe --verbose tcp://192.168.0.9
virsh migrate win7_kvm_nfs --live qemu+ssh://192.168.0.8/system --unsafe --verbose tcp://192.168.0.8

127 <==> 153
virsh migrate win7_nfs_kvm --live qemu+ssh://10.74.120.153/system --unsafe --verbose
virsh migrate win7_nfs_kvm --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_nfs_kvm --live qemu+ssh://10.74.120.153/system --unsafe --verbose
virsh migrate rhel_nfs_kvm --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate cgslv5_kvm_nfs --live qemu+ssh://10.74.120.153/system --unsafe --verbose
virsh migrate cgslv5_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

105 <==> 106
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.106/system --unsafe --verbose
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.106/system --unsafe --verbose
#XBEZLE
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose --compressed

#多线程压缩迁移
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose --comp-methods mt --comp-mt-level 1 --comp-mt-threads 12 --comp-mt-dthreads 3

#libvirt中调试MT迁移功能需要的断点
b qemuMonitorJSONSetMigrationCompression
b qemu_monitor_json.c:297
p cmdstr
b qemuMonitorJSONSetMigrationCapability if capability==QEMU_MONITOR_MIGRATION_CAPS_COMPRESS
b qemuMigrationCompressionParse
b qemuMigrationCompressMethodTypeFromString

#有一定概率会迁移失败：
error: operation failed: migration job: unexpectedly failed

########################################################################
# 通过QEMU命令行迁移
########################################################################

########################################################################
#同机之间迁移 192.168.2.106
########################################################################
源:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
目标:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:1 -monitor stdio -incoming tcp:0:6666

在源中启动迁移，并查看log
(qemu) migrate -d tcp:0:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off
Migration status: active
total time: 4831 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 146510 kbytes
throughput: 268.70 mbps
remaining ram: 186004 kbytes
total ram: 2114376 kbytes
duplicate: 446516 pages
skipped: 0 pages
normal: 35577 pages
normal bytes: 142308 kbytes
dirty sync count: 1
(qemu) quit
迁移完成后，可以通过192.168.2.106:5901连接被迁移后的虚拟机

########################################################################
#不同机之间迁移 192.168.2.106 ==> 192.168.2.105
########################################################################
源:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
目标:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.105:0 -monitor stdio -incoming tcp:192.168.2.105:6666

在源中启动迁移，并查看log
(qemu) migrate -d tcp:192.168.2.105:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off
Migration status: active
total time: 4831 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 146510 kbytes
throughput: 268.70 mbps
remaining ram: 186004 kbytes
total ram: 2114376 kbytes
duplicate: 446516 pages
skipped: 0 pages
normal: 35577 pages
normal bytes: 142308 kbytes
dirty sync count: 1
(qemu) quit
迁移完成后，可以通过192.168.2.105:5900连接被迁移后的虚拟机

########################################################################
#多线程压缩不同机之间迁移 192.168.2.106 ==> 192.168.2.105
########################################################################
源：
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
QEMU 2.5.1 monitor - type 'help' for more information
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off
(qemu) migrate_set_capability compress on
(qemu) migrate_set_parameter compress-threads 12
(qemu) migrate_set_parameter compress-level 1
(qemu) migrate -d tcp:192.168.2.105:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: on events: off x-postcopy-ram: off
Migration status: active
total time: 3274 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 100655 kbytes
throughput: 270.25 mbps
remaining ram: 170208 kbytes
total ram: 2114376 kbytes
duplicate: 445720 pages
skipped: 0 pages
normal: 40322 pages
normal bytes: 161288 kbytes
dirty sync count: 1

目标
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.105:0 -monitor stdio -incoming tcp:192.168.2.105:6666
QEMU 2.5.1 monitor - type 'help' for more information
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off
(qemu) migrate_set_capability compress on
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: on events: off x-postcopy-ram: off
(qemu) migrate_set_parameter compress-level 1
(qemu) migrate_set_parameter decompress-threads 3
(qemu) info migrate
(qemu) quit

#####################################################
# 网络加压力
#####################################################
# 源端
iperf3 -s
# 目的端

#NTP同步

ntpdate 10.74.120.172

watch -n 0.1 date
#############################################
# NETPERF
#############################################
#NETPERF 编译
tar jxvf netperf-2.6.0.tar.bz2
cd netperf-2.6.0/
./configure --prefix /home/liufeng/netperf
make && make install

#运行netperf

./netperf -H 10.74.120.9 -l 6000000
./netperf -H 10.74.120.153 -l 6000000
./netserver &

################################################
# QEMU
##################################################
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug
#标准编译
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl

# 加入NUMA的支持
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl --enable-numa

#加入调试信息
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info

#使能bzip2压缩需要sudo yum install bzip2-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-bzip2
#使能SNAPPY压缩需要 sudo yum install snappy-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-bzip2 --enable-debug-info --enable-snappy
#使能RDMA 需要sudo yum install librdmacm-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-rdma
#使能LZO压缩需要sudo yum install lzo-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-lzo

#使能profiler
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-profiler

#使能Document 需要sudo yum install texinfo.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-docs

#使能加密需要 sudo yum install libgcrypt-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-gcrypt

./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --disable-docs --disable-werror

make -j 64
sudo make install

make && make install

g_ltp_migration_s
LTP_DBGDUMP
g_slew_for_migrate_switch
if (1 == g_ltp_migration_s.ul_slew_switch
|| s->lost_tick_policy == LOST_TICK_POLICY_SLEW) {

####################################
# QEMU RPM
#####################################
cd ../SOURCES
tar -zcf qemu-2.5.0.tar.gz qemu-2.5.0
cd ../SPECS/
rpmbuild -bb qemu.spec --without check --define "_topdir `pwd`/../"

################################################
# QGA
##################################################
qemu-ga -p /dev/virtio-ports/org.qemu.guest_agent.0 &
virsh qemu-agent-command 12 '{"execute":"guest-get-time"}'
virsh qemu-agent-command 12 '{"execute":"guest-set-time", "arguments":{"time":1416970257039676000}}'

#####################################################
# 编译CGELX
#####################################################
# 4X
cd /home/liufeng/workspace/src/CGEL_V4.01.20_kernel/src/target/build/cgel4.0/product/build
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu- menuconfig
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu- all j=16

# 5X
cd /home/liufeng/workspace/src/CGEL_V5.x_kernel/src/target/build/cgel5.0/product/build
make ARCH=x86 LSPSRC=../../../../bsp/cgel5.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu- menuconfig
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu- all j=16

################################################################################################
# 制作ramdisk
################################################################################################
cd /home/kvm/disks
dd if=/dev/zero of=initrd.img bs=1k count=4194304
mke2fs -F -v -m0 initrd.img
mkdir initrd
mount -o loop initrd.img initrd
#gzip -f -9 initrd.img
#mkimage -T ramdisk -C gzip -n 'CHM3 Ramdisk Image' -d initrd.img.gz ramdisk.bin
#/home/liufeng/workspace/tools/mkimage -T ramdisk -C gzip -n 'CHM3 Ramdisk Image' -d initrd.img.gz ramdisk.bin
#mv ramdisk.bin rootfs.gz

##################################################################################################
# rpmbuld方式libvirt编译
##################################################################################################
#10.74.120.145
cd /home/code/libvirt/SOURCES
tar -zcf libvirt-1.2.1.tar.gz libvirt-1.2.1
tar -zcf libvirt-1.3.3.tar.gz libvirt-1.3.3
cd ../SPECS/
rpmbuild -bb libvirt.spec --without check --define "_topdir `pwd`/../"
pushd /home/code/libvirt/RPMS/x86_64/
scp *.rpm root@10.74.121.127:/home/wyc

scp *.rpm root@10.74.120.247:/home/liufeng/libvirt

##################################################################################################
# 手动编译安装libvirt
##################################################################################################
sudo yum install perl-CPAN.noarch
perl -MCPAN -e shell
install Digest::MD5.pm
install XML::XPath
#第一次修改autogen.sh如下
vim autogen.sh
<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#if test -z "$*" && test -z "$EXTRA_ARGS" && test -f config.status; then
# ./config.status --recheck
#else
# $srcdir/configure $EXTRA_ARGS "$@"
#fi && {
echo
echo "Now type 'make' to compile libvirt."
#}
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
./autogen.sh
./configure --prefix=/home/liufeng/workspace/tools/libvirt --without-xen --without-openvz --without-vbox --without-libxl --without-xenapi --without-uml --without-phyp --without-hyperv --without-vmware --without-parallels --without-storage-rbd --without-storage-sheepdog --without-storage-gluster --without-apparmor --without-hal --with-firewalld '--with-packager=Unknown, 2016-04-19-12:26:41, dev106' --with-packager-version=1.el7.centos --with-qemu-user=qemu --with-qemu-group=qemu --enable-werror --enable-expensive-tests --with-init_script=systemd
make
make install

或者注释掉autobuild.sh的make install后，先运行一下，就可以用libvirt.spce了

#10.74.120.145
cd /home/code/libvirt/SOURCES
tar -zcf libvirt-1.2.1.tar.gz libvirt-1.2.1
cd ../SPECS/
rpmbuild -bb libvirt.spec --without check --define "_topdir `pwd`/../"
pushd /home/code/libvirt/RPMS/x86_64/
scp *.rpm root@10.74.121.127:/home/wyc

scp *.rpm root@10.74.120.247:/home/liufeng/libvirt

#10.74.121.127
pushd /home/wyc
rm -rf *lock*
rpm -ivh * --force
systemctl daemon-reload
service libvirtd start
service libvirtd status
service libvirtd stop

##################################################################################################
# init编译
##################################################################################################
cd /home/liufeng/workspace/src/ZXCCP-VPlat_CV/trunk/source.repo/src/v5.1/init/SOURCES
tar -zcf init-1.tar.gz init-1
cd ../SPECS/
rpmbuild -bb init.spec --define "_topdir `pwd`/../"
rpmbuild -bb /home/SVN/init/init/SPECS/init.spec

rpm -ivh * --force

##################################################################################################
# LVM管理
##################################################################################################

#启动并发现磁阵
[root@lf-127 ~]# iscsiadm -m discovery -t sendtargets -p 10.74.120.45
10.74.120.45:3260,1 vplat_liufeng_sdg5.zte.disk --> 作为第二条命令的输出
[root@lf-127 ~]# iscsiadm -m node -T vplat_liufeng_sdg5.zte.disk -l
[root@lf-127 ~]# vgscan
Reading all physical volumes. This may take a while...
Found volume group "zxvei" using metadata type lvm2
Found volume group "ZXVE-liufeng-vg" using metadata type lvm2
[root@lf-127 ~]# vgs
VG #PV #LV #SN Attr VSize VFree
ZXVE-liufeng-vg 1 1 0 wz--n- 298.02g 288.02g
zxvei 1 3 0 wz--n- 278.91g 0
[root@lf-127 ~]#

#查看有哪些vgs
vgs

#查看已经创建了哪些磁盘
lvs

#创建磁盘
lvcreate -L 80G -n build ZXVE-liufeng-vg
lvcreate -L 100G -n tarball ZXVE-liufeng-vg

#挂载分区
mkfs -t ext3 /dev/ZXVE-liufeng-vg/build
mkfs -t ext3 /dev/ZXVE-liufeng-vg/tarball
mount /dev/ZXVE-liufeng-vg/build /home/build/
mount /dev/ZXVE-liufeng-vg/tarball /home/liufeng/tarball

#删除磁盘
lvremove /dev/ZXVE-liufeng-vg/build
lvremove /dev/ZXVE-liufeng-vg/tarball
#

##################################################################################################
# 用dd建立磁盘
##################################################################################################
# 制作2G大文件
dd if=/dev/zero of=initrd.img bs=1k count=2097152

##################################################################################################
# 清理内存
##################################################################################################
清空page_cache
echo 3 > /proc/sys/vm/drop_caches

NFS服务器配置
rpm -qa|grep nfs : 查看nfs安装情况
service rpcbind start
systemctl enable nfs-server.service
service nfs start ：启动nfs
systemctl enable nfs-server.service

vi /etc/exports   : 编辑配置文件
/opt/vmpool 10.74.169.0/24(rw)
service nfs restart   : 重启服务
showmount -e/a       : 默认查看自己共享的服务/已经与客户端连接上的目录信息
service nfs status
service nfs stop
注意事项：虚拟机的镜像文件和配置文件需要修改权限为所有权限。chmod 777 *.img/*.xml
共享目录需要设置为777权限

客户端
showmount -e 10.74.169.162（NFS服务器IP）
#mount 10.74.169.162:/opt/vmpool /opt/vmpool
# 解决mount.nfs: an incorrect mount option was specified的问题
mount -t nfs -o nolock,nfsvers=3,rw 192.168.2.106:/root/tmp /root/tmp
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/root/tmp /root/tmp
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/home/liufeng/workspace/vm /home/liufeng/workspace/vm
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/home/kvm/vm /home/kvm/vm

mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/kvm/vm /home/kvm/vm

mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/kvm/vm /home/kvm/vm

mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/kvm/vm /home/kvm/vm

mount过来的路径应该是一致的。
mount |grep nfs

##################################################################################################
# KVM4NFV编译
##################################################################################################
# RT内核
# BIOS中使能SR-IOV, X2APIC
# 内核配合中使能 CONFIG_X86_X2APIC = y
cd /home/liufeng/workspace/src/open/kvmfornfv
patch -p1 < patches/kernel/0001-Restore-deprecated-XFS-API.patch
cd kernel
cp arch/x86/configs/opnfv.config .config
make oldconfig
make all -j 8
make modules
vim /etc/grub2.cfg
vim /etc/default/grub
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
GRUB_DEFAULT=0
#GRUB_TIMEOUT_STYLE=hidden
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"
GRUB_CMDLINE_LINUX_DEFAULT="text"
GRUB_CMDLINE_LINUX=""
GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 4.15.18-feng"

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
sudo update-grub
#有时候需要bzImage
make bzImage
sudo make modules_install install

##################################################################################################
# 路由转发功能
##################################################################################################
echo 1 > /proc/sys/net/ipv4/ip_forward

##################################################################################################
# Namespace
##################################################################################################
ip netns add foo
ip netns
ip netns exec foo ip a
ip netns exec foo ip link set lo up
ip netns exec foo ip a
ip netns exec foo route
ip netns exec foo iptables -nL
ip netns exec foo ping 192.168.1.163
ip link add tap-foo type veth peer name tap-root
ip a
ethtool -S tap-root
ethtool -S tap-foo
ip link set tap-foo netns foo
ip a
ip netns exec foo ip a
ip netns exec foo ip addr add 192.168.1.167/24 dev tap-foo
ip netns exec foo ip link set tap-foo up
ip netns exec foo ip a
ip addr add 192.168.1.168/24 dev tap-root
ip link set tap-root up
ip a
ping 192.168.1.167

ip netns exec foo ip addr del 192.168.1.167/24 dev tap-foo
ip netns exec foo ip addr add 10.0.0.1/24 dev tap-foo
ip addr del 192.168.1.168/24 dev tap-root
ip addr add 10.0.0.2/24 dev tap-root

##################################################################################################
# 通过LIBVIRT建立 openvswitch 的网络交换
##################################################################################################
virsh net-destroy default
virsh net-list --all
virsh net-autostart --disable default
virsh net-autostart --network default
virsh net-undefine default

ovs-vsctl add-br br-int
vim libvirt-vlans.xml
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
<network>
<name>lf-ovs-net1</name>
<forward mode='bridge'/>
<bridge name='br-int'/>
<virtualport type='openvswitch'/>
<portgroup name='no-vlan' default='yes'>
</portgroup>
<portgroup name='vlan-100'>
<vlan>
<tag id='100'/>
</vlan>
</portgroup>
<portgroup name='vlan-200'>
<vlan>
<tag id='200'/>
</vlan>
</portgroup>
</network>
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
virsh net-define libvirt-vlans.xml
virsh net-list --all
virsh net-start lf-ovs-net1
virsh net-list --all

vim centos.xml
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
<interface type='network'>
<source network='lf-ovs-net1' portgroup='vlan-100'/>
<model type='virtio'/>
</interface>
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

virsh define centos.xml
virsh list --all
virsh start centos_kvm_vm

ovs-vsctl show

ip link add br-int-tap100 type veth peer name tap100
ip link
ovs-vsctl add-port br-int br-int-tap100
ovs-vsctl set port br-int-tap100 tag=100
ip addr add 192.168.2.21/24 dev tap100
ip link set tap100 up
ip link set br-int-tap100 up
ovs-vsctl show
ip a
ping 192.168.2.20

#VM内部设置
ip addr add 192.168.2.20/24 dev eth0

##################################################################################################
# OVS环境搭建(注意要先编译安装OVS，再编译安装libvirt，否则自动configure的时候里面的ovs支持功能没有打开)
##################################################################################################

git clone https://github.com/openvswitch/ovs.git
git checkout -b compile

./boot.sh
./configure --prefix=/home/kvm/tools/ovs --with-linux=/lib/modules/`uname -r`/build
make
make install

# 第一次
ifconfig enp5s0f3 0 up
modprobe openvswitch
ovsdb-tool create /home/kvm/tools/ovs/etc/openvswitch/conf.db
ovsdb-server --remote=punix:/home/kvm/tools/ovs/var/run/openvswitch/db.sock --pidfile --detach
ovs-vsctl --no-wait init
ovs-vswitchd --pidfile --detach
ps aux | grep ovs
ovs-vsctl show
ovs-vsctl add-br br1
ovs-vsctl add-port br1 ens2f0
#ovs-vsctl add-port br1 enp5s0f3
#ovs-vsctl del-port br1 enp5s0f3
ovs-vsctl show

#以后
modprobe openvswitch
ovsdb-server --remote=punix:/home/kvm/tools/ovs/var/run/openvswitch/db.sock --pidfile --detach --log-file --verbose
ovs-vswitchd --pidfile --detach
ps aux | grep ovs
ovs-vsctl show

# OpenFlow相关问题
ovs-ofctl -O openflow13 show br-int
ovs-ofctl -O openflow13 dump-flows br-int

# 通过veth口连接两个OVS的bridge
ovs-vsctl add-br br2
ovs-vsctl add-br br3
ip link add name veth2 type veth peer name veth3
ovs-vsctl add-port br2 veth2
ovs-vsctl add-port br3 veth3
ovs-vsctl show

# 通过patch口连接两个OVS的bridge
ovs-vsctl add-br br5
ovs-vsctl add-br br6
ovs-vsctl add-port br5 patch-to-br5
ovs-vsctl set interface patch-to-br5 type=patch
ovs-vsctl set interface patch-to-br5 options:peer=patch-to-br6
ovs-vsctl add-port br6 patch-to-br6
ovs-vsctl set interface patch-to-br6 type=patch
ovs-vsctl set interface patch-to-br6 options:peer=patch-to-br5
ovs-vsctl show

ovs-vsctl show

ip link add br5-tap5 type veth peer name tap5
ip link
ovs-vsctl add-port br5 br5-tap5
ovs-vsctl set port br5-tap5 tag=5
ip addr add 192.168.2.21/24 dev tap5
ip link set tap5 up
ip link set br5-tap5 up
ovs-vsctl show

ip link add br6-tap6 type veth peer name tap6
ip link
ovs-vsctl add-port br6 br6-tap6
ovs-vsctl set port br6-tap6 tag=6
ip addr add 192.168.2.22/24 dev tap6
ip link set tap6 up
ip link set br6-tap6 up
ovs-vsctl show

######################################################################################################
# 多网卡绑定与建桥
######################################################################################################
cat /etc/modprobe.d/nicbond.conf
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
alias bond0 bonding
options bond0 mode=1 miimon=100
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

modprobe bonding

cd /etc/sysconfig/network-scripts/
cat ifcfg-bond0
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
DEVICE=bond0
ONBOOT=yes
BRIDGE=br-bond
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

cat ifcfg-enp5s0f0
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
HWADDR=00:1B:21:08:11:76
TYPE=Ethernet
BOOTPROTO=none
NAME=enp5s0f0
UUID=f180b55f-85ba-4d1b-a6e5-c08bea077af4
ONBOOT=yes
MASTER=bond0
SLAVE=yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

cat ifcfg-enp5s0f1
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
HWADDR=00:1B:21:08:11:77
TYPE=Ethernet
BOOTPROTO=none
NAME=enp5s0f1
ONBOOT=yes
MASTER=bond0
SLAVE=yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

cat ifcfg-br-bond
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
DEVICE=br-bond
TYPE=Bridge
ONBOOT=yes
IPADDR=192.168.1.161
NETMASK=255.255.255.0
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

ifconfig br-bond
cat /proc/net/bonding/bond0
ping 192.168.1.162

######################################################################################################
# PASSTHROUGH
######################################################################################################
# 修改grub
cat /etc/grub2.cfg
menuentry 'CentOS Linux (4.4.6-rt14nfv-dev162) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-123.el7.x86_64-advanced-5ff43bc4-a100-4f76-8e3b-7b6b7ed6b9cc' {
......
linux16 /vmlinuz-4.4.6-rt14nfv-dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 intel_iommu=on
initrd16 /initramfs-4.4.6-rt14nfv-dev162.img
}
menuentry 'CentOS Linux (4.6.0-rc7-dev162) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-123.el7.x86_64-advanced-5ff43bc4-a100-4f76-8e3b-7b6b7ed6b9cc' {
......
linux16 /vmlinuz-4.6.0-rc7-dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 intel_iommu=on
initrd16 /initramfs-4.6.0-rc7-dev162.img
}

ls /sys/bus/pci/drivers/pci-stub/

dmesg | grep enp5s0f3
#[ 5.311049] igb 0000:05:00.3 enp5s0f3: renamed from eth1

dmesg | grep eth1
#[ 5.075402] igb 0000:05:00.3: added PHC on eth1

lspci -D -s 05:00.3
lspci -Dn -s 05:00.3

echo -n "8086 1521" > /sys/bus/pci/drivers/pci-stub/new_id
echo 0000:05:00.3 > /sys/bus/pci/devices/0000\:05\:00.3/driver/unbind
echo 0000:05:00.3 > /sys/bus/pci/drivers/pci-stub/bind

lspci -k -s 05:00.3

cd /home/liufeng/scripts/kvm
./pcistub.sh -h 05:00.3
qemu-system-x86_64 -machine accel=kvm -smp 8 -drive file=/home/kvm/vm/disk/centos.qcow2 -m 2048 -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait -device kvm-pci-assign,host=05:00.3,id=lfdev0,addr=0x8 -net none -boot d

qemu-system-x86_64 -machine accel=kvm -smp 8 -drive file=/home/kvm/vm/disk/centos.qcow2 -m 2048 -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait -device kvm-pci-assign,host=01:00.3,id=lfdev0,addr=0x8 -net none -boot d

######################################################################################################
# SR-IOV
######################################################################################################

lspci -v -s 05:00.3
modinfo igb
modinfo ixgbe
lspci | grep Eth
modprobe -r igb
modprobe igb max_vfx=7

modprobe -r ixgbe
#modprobe ixgbe max_vfs=63
modprobe ixgbe max_vfs=2

cat /etc/modprobe.d/igb.conf
option igb max_vfs=7
option ixgbe max_vfx=63

ls -l /sys/bus/pci/devices/0000\:81\:00.0/virtfn*

######################################################################################################
# DPDK
######################################################################################################
# 内核配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
UIO = m ===> uio.ko
HUGETLBFS
PROC_PAGE_MONITOR
HPET
HPET_MMAP
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

# DPDK
wget http://fast.dpdk.org/rel/dpdk-16.07.tar.xz
xz -d dpdk-16.07.tar.xz
tar xvf dpdk-16.07.tar
export DPDK_DIR=/home/liufeng/workspace/src/open/dpdk-16.07
cd $DPDK_DIR

export DPDK_TARGET=x86_64-native-linuxapp-gcc
#export DPDK_TARGET=x86_64-ivshmem-linuxapp-gcc
export DPDK_BUILD=$DPDK_DIR/$DPDK_TARGET
make install T=$DPDK_TARGET DESTDIR=install

#OVS-DPDK
git clone https://github.com/openvswitch/ovs.git
export OVS_DIR=/home/liufeng/workspace/src/open/ovs
#./configure --prefix=/home/kvm/tools/ovs --with-linux=/lib/modules/`uname -r`/build
./boot.sh
./configure --prefix=/home/kvm/tools/ovs-dpdk --with-dpdk=$DPDK_BUILD
make -j 8
make install

# 巨页设置
echo 'vm.nr_hugepages=2048' > /etc/sysctl.d/hugepages.conf
sysctl -w vm.nr_hugepages=1024
grep HugePages_ /proc/meminfo
mount -t hugetlbfs none /dev/hugepages

#VFIO设置
dmesg | grep -e DMAR -e IOMMU
cat /etc/grub2.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
linux16 /vmlinuz-4.6.0-rc7-compile@dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 iommu=pt intel_iommu=on
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
cat /proc/cmdline
modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*
$DPDK_DIR/tools/dpdk-devbind.py --bind=vfio-pci enp5s0f3
$DPDK_DIR/tools/dpdk-devbind.py --status

#启动OVS-DPDK
modprobe openvswitch
mkdir -p /home/kvm/tools/ovs-dpdk/etc/openvswitch/
mkdir -p /home/kvm/tools/ovs-dpdk/var/run/openvswitch/
rm /home/kvm/tools/ovs-dpdk/etc/openvswitch/conf.db
ovsdb-tool create /home/kvm/tools/ovs-dpdk/etc/openvswitch/conf.db /home/kvm/tools/ovs-dpdk/share/openvswitch/vswitch.ovsschema
ovsdb-server --remote=punix:/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init
# SSL
# ovsdb-server --remote=punix:/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --private-key=db:Open_vSwitch,SSL,private_key --certificate=Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --pidfile --detach

export DB_SOCK=/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file --verbose
ovs-vswitchd --dpdk unix:$DB_SOCK --pidfile --detach --log-file

ovs-vsctl add-br br1 -- set bridge br1 datapath_type=netdev
ovs-vsctl add-port br1 dpdk0 -- set Interface dpdk0 type=dpdk
ovs-vsctl add-port br1 dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser
ovs-vsctl add-port br1 dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser
ovs-vsctl show

top -H
ps -eLo pid,psr,comm | grep pmd

ovs-appctl dpif-netdev/pmd-stats-show
ovs-appctl dpif-netdev/pmd-stats-clear

ovs-appctl dpif-netdev/pmd-rxq-show

export VM_NAME=Centos-vm
export GUEST_MEM=1024M
export QCOW2_IMAGE=/home/kvm/vm/disk/centos.qcow2
export VHOST_SOCK_DIR=/home/kvm/tools/ovs-dpdk/var/run/openvswitch/
qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off --nographic -snapshot

qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait

qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait

ovs-appctl -t ovs-vswitchd exit
ovs-appctl -t ovsdb-server exit
ovs-vsctl del-br br1

#ovs-ofctl del-flows br1
#ovs-ofctl add-flow br0 in_port=1,action=output:2

mkdir -p /dev/hugepages/libvirt/qemu
virsh create centos-ovs-dpdk.xml

#####################################################
# QEMU调试启动
#####################################################
set var g_stop_here=0

#####################################################
# 配置服务器NAT
#####################################################
iptables -t nat -A PREROUTING -d 172.16.0.3 -p tcp --dport 80 -j DNAT --to 10.100.17.183:80
iptables -t nat -A POSTROUTING -d 10.100.17.183 -p tcp --dport 80 -j SNAT --to 172.16.0.3

#####################################################
# OVS-DPDK 的配置
#####################################################

echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages

cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages

cd /etc/
mkdir dpdk
touch dpdk.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
NR_2M_PAGES=1024
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
systemctl start dpdk
mount | grep huge
#cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
#hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
#hugetlbfs on /mnt/hugepages type hugetlbfs (rw,relatime)
ls /dev/hugepages/
systemctl start openvswitch
ps -ef | grep ovs
ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl get Open_vSwitch . other_config
systemctl restart openvswitch
ps -eLo comm,psr,pid | grep {pid of ovs-vswitchd }
#ovs-vswitchd 0 10623
#eal-intr-thread 0 10623
#dpdk_watchdog1 13 10623
#vhost_thread2 32 10623
#urcu3 29 10623

ovs-vsctl add-br br-test -- set bridge br-test datapath_type=netdev
ovs-vsctl show
ovs-vsctl add-port br-test vhost1 -- set interface vhost1 type=dpdkvhostuser
ovs-vsctl add-port br-test vhost2 -- set interface vhost2 type=dpdkvhostuser
ovs-vsctl show

# 两个DPDK 的port的socket文件分别在如下位置
#/var/run/openvswitch/vhost1
#/var/run/openvswitch/vhost2

# 调试命令
ovs-ofctl show br-test
ovs-ofctl dump-ports br-test

###########################################################################################
# SSH免密码登陆
###########################################################################################
# HOST A （SRC）
ssh-keygen -t rsa -P ''
scp .ssh/id_rsa.pub root@10.20.2.2:/tmp
# HOST B (DEST)
ssh 10.20.2.2
cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys
# 从A登陆B，免除密码

###########################################################################################
# 安装 EPEL 扩展库 Extra Packages for Enterprise
###########################################################################################
sudo yum install epel-release.noarch
yum repolist

######################################################################################################
# 下载 CentOS 内核
######################################################################################################

# 修改/etc/CentOS-Sources.repo, enable修改为1
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[base-source]
name=CentOS-$releasever - Base Sources
baseurl=http://vault.centos.org/centos/$releasever/os/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#released updates
[updates-source]
name=CentOS-$releasever - Updates Sources
baseurl=http://vault.centos.org/centos/$releasever/updates/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that may be useful
[extras-source]
name=CentOS-$releasever - Extras Sources
baseurl=http://vault.centos.org/centos/$releasever/extras/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that extend functionality of existing packages
[centosplus-source]
name=CentOS-$releasever - Plus Sources
baseurl=http://vault.centos.org/centos/$releasever/centosplus/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

yum clean all
yum repolist
yumdownloader --source kernel

###########################################################################################
# 设置内核在线CPU数量和内存数量的内核参数
###########################################################################################
# 动态设置
# 下线CPU1
echo 0 > /sys/devices/system/cpu/cpu1/online
# 上线CPU1
echo 1 > /sys/devices/system/cpu/cpu1/online

# 上线10个CPU
nr_cpus=10

###########################################################################################
# 使用mpstat 和 iostat 进行性能监控
###########################################################################################
yum install sysstat.x86_64

###########################################################################################
# RT补丁应用
###########################################################################################
cd linux-4.4.79
patch -p1 < patch-4.4.79-rt92.patch

################################################################################
# 增加IPMI WATCHDOG
################################################################################
modprobe ipmi_si
modprobe ipmi_watchdog timeout=90 action=reset pretimeout=30 preaction=pre_int preop=preop_panic

################################################################################
# 使libvir启动的qemu-kvm在qemu崩溃后可以产生 core dump
################################################################################
在/etc/libvirt/qemu.conf最尾部添加如下内容
dump_image_format = "raw"
auto_dump_path = "/var/lib/libvirt/qemu/dump"
max_core = "unlimited"

然后使用如下命令重启libvirt：
service libvirtd restart
service libvirtd status

重启所有vm后，即可产生coredump

################################################################################
# CentOS 安装 tunctl
################################################################################
/etc/yum.repos.d/nux-misc.repo
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[nux-misc]
name=Nux Misc
baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/
enabled=0
gpgcheck=1
gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
yum --enablerepo=nux-misc install tunctl

################################################################################
# Ubuntu 安装配置 SAMBA
################################################################################

/etc/samba/smb.conf
#################################################

[workspace]
path = /home/liufeng/workspace
available = yes
browseable = yes
public = yes
writable = yes
###################################################

sudo smbpasswd -a liufeng

sudo service smbd status

################################################################################
# Ubuntu 删除安装包
################################################################################
dpkg -l | grep vpp

################################################################################
# OVS DPDK 安装
################################################################################

vim .bash_profile
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

MYTOOLS=/home/liufeng/workspace/tools
export MYTOOLS

PATH=$MYTOOLS/bin:$MYTOOLS/sbin:$MYTOOLS/libexec:$PATH:$HOME/.local/bin:$HOME/bin
export PATH

C_INCLUDE_PATH=$MYTOOLS/include:$C_INCLUDE_PATH
export C_INCLUDE_PATH

LD_LIBRARY_PATH=$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
source ~/.bash_profile

sudo vim /etc/grub2.cfg
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
cat /proc/cmdline

yum install "@Development Tools" automake kernel-tools openssl-devel python-devel python-six checkpolicy selinux-policy-devel bash-completion
yum install kernel-devel-3.10.0-1127.19.1.el7.x86_64

cd /home/liufeng/workspace/src/open
git clone git@github.com:DPDK/dpdk.git
git clone git@github.com:openvswitch/ovs.git

cd /home/liufeng/workspace/src/open/dpdk
git checkout v19.11 -b v19.11
make config T=x86_64-native-linuxapp-gcc
mkdir /home/liufeng/workspace/src/open/dpdk/sdk
make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/open/dpdk/sdk
#sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr
sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/

cd /home/liufeng/workspace/src/open/ovs
#git checkout v2.12.0 -b v2.12.0
git checkout v2.13.0 -b v2.13.0

##sed -i "s/ ETHER_/ RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/(ETHER_/(RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/ ETHER_/ RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/ e_RTE_METER_/ RTE_COLOR_/" lib/netdev-dpdk.c
##sed -i "s/struct ether_addr/struct rte_ether_addr/" lib/netdev-dpdk.c
##sed -i "s/struct ether_hdr/struct rte_ether_hdr/" lib/netdev-dpdk.c
#./boot.sh

#./configure \
--with-dpdk=/home/liufeng/workspace/src/open/dpdk/sdk \
--prefix=/home/liufeng/workspace/tools/ \
--exec-prefix=/home/liufeng/workspace/tools/ \
--sysconfdir=/home/liufeng/workspace/tools/etc \
--localstatedir=/home/liufeng/workspace/tools/var

#make
#sudo make install

sudo modprobe vfio
sudo modprobe vfio-pci
sudo modprobe uio_pci_generic
sudo insmod /home/liufeng/workspace/src/open/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko

dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down

sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3

dpdk-devbind --status

su
grep HugePages_ /proc/meminfo
#echo 1024 > /proc/sys/vm/nr_hugepages
#echo 'vm.nr_hugepages=1024' > /etc/sysctl.d/hugepages.conf
mount -t hugetlbfs none /dev/hugepages

#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/

#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db

# 启动ovsdb server
#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema

sudo /home/liufeng/workspace/tools/sbin/ovsdb-server /home/liufeng/workspace/tools/etc/openvswitch/conf.db \
-vconsole:emer -vsyslog:err -vfile:info \
--remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
--private-key=db:Open_vSwitch,SSL,private_key \
--certificate=db:Open_vSwitch,SSL,certificate \
--bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \
--detach --monitor

# 初始化dpdk
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait init
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 指定的sockets从hugepages预先分配的内存
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0"
# 指定在某些core上运行
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x02

#查看自定义的dpdk参数
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-socket-mem
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:pmd-cpu-mask
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-init

#启动vswitchd进程
sudo /home/liufeng/workspace/tools/sbin/ovs-vswitchd unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
-vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \
--detach --monitor

#创建openvswitch网桥
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev

#把dpdk端口加入网桥
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1

#chmod 0666 /dev/vfio -R
#readlink /sys/bus/pci/devices/0000:01:00.0/iommu_group
#readlink /sys/bus/pci/devices/0000:01:00.1/iommu_group

#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set-controller br0 tcp:10.111.67.173:6633

/home/liufeng/workspace/tools/bin/ovs-vsctl show

sudo /home/liufeng/workspace/tools/bin/ovs-ofctl dump-flows br0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0

sudo /home/liufeng/workspace/tools/bin/ovsdb-client dump

################################################################################
# OVS DPDK VM 安装
################################################################################

#vim ~/.bash_profile
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#DPDK_SDK=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#export DPDK_SDK
#
#LD_LIBRARY_PATH=$DPDK_SDK/lib:$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH
#export LD_LIBRARY_PATH
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#source ~/.bash_profile
#
#
#sudo vim /etc/grub2.cfg
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=6
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#
#
#/home/liufeng/workspace/src/ovs-dpdk
#git clone git@github.com:DPDK/dpdk.git
#git clone git@github.com:openvswitch/ovs.git
#
#
#cd /home/liufeng/workspace/src/ovs-dpdk/dpdk
#git checkout v19.11 -b v19.11
#vim config/common_base
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#CONFIG_RTE_BUILD_SHARED_LIB=y
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

#make config T=x86_64-native-linuxapp-gcc
#sed -ri 's,(PMD_PCAP=).*,\1y,' build/.config
#mkdir /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
#make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk
##sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr
#sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/
#
#vim /etc/ld.so.conf.d/dpdk.conf
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#ldconfig
#
#sudo yum install autoconf.noarch automake.noarch libtool.x86_64 python3-pip.noarch

#cd /home/liufeng/workspace/src/open/ovs
#git checkout v2.13.0 -b v2.13.0
#./boot.sh
#./configure \
#--with-dpdk=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk \
#--prefix=/home/liufeng/workspace/tools/ \
#--exec-prefix=/home/liufeng/workspace/tools/ \
#--sysconfdir=/home/liufeng/workspace/tools/etc \
#--localstatedir=/home/liufeng/workspace/tools/var

#make
#sudo make install

sudo modprobe vfio
sudo modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*

#sudo modprobe uio_pci_generic
#sudo insmod /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko

dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down

sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3

dpdk-devbind --status

mount -t hugetlbfs -o pagesize=1G none /dev/hugepages
grep HugePages_ /proc/meminfo

#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/

#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db

#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema

# 启动ovsdb server
ovsdb-server \
/home/liufeng/workspace/tools/etc/openvswitch/conf.db \
-vconsole:emer -vsyslog:err -vfile:info \
--remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
--remote=db:Open_vSwitch,Open_vSwitch,manager_options \
--private-key=db:Open_vSwitch,SSL,private_key \
--certificate=db:Open_vSwitch,SSL,certificate \
--bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \
--detach --monitor

ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true

#启动vswitchd进程
ovs-vswitchd \
unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
-vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \
--detach --monitor

top -H
ps -eLo pid,psr,comm | grep pmd

ovs-vsctl --version
ovs-vsctl show
#把dpdk端口加入网桥
#ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
#ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0
#ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1
#ovs-vsctl add-port br0 vhost-user-1 -- set Interface vhost-user-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0"
#ovs-vsctl add-port br0 vhost-user-1-1 -- set Interface vhost-user-1-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0-1"
#ovs-vsctl add-port br0 vhost-user-2 -- set Interface vhost-user-2 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock2"

ovs-ofctl dump-flows br0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0

ovsdb-client dump

#git clone https://git.qemu.org/git/qemu.git
#/home/liufeng/workspace/src/ovs-dpdk/qemu
#git checkout v3.0.0-rc4 -b v3.0.0-rc4
#./configure --prefix=/home/liufeng/workspace/tools/ --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl --disable-git-update
#scripts/git-submodule.sh update ui/keycodemapdb dtc
#make
#sudo make install

#pushd /home/kvm/disk
#qemu-img create -f qcow2 -o size=50G vm3.img
#popd

#vim /etc/yum.repos.d/nux-misc.repo
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#[nux-misc]
#name=Nux Misc
#baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/
#enabled=0
#gpgcheck=1
#gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#yum --enablerepo=nux-misc install tunctl

#建立VM网络
tunctl -t tap30
tunctl -t tap33
brctl addif br1 tap30
brctl addif br1 tap33
brctl show
ifconfig tap30 promisc up
ifconfig tap33 promisc up

#安装：
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 -boot d \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38

# 普通启动 VM3
#qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 \
#-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \
#-vnc 0.0.0.0:30 -monitor stdio \
#-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38

# 查看OVS状态
tailf /home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log

# vhost user 启动 VM0
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 \
-chardev socket,id=char1,path=/tmp/sock0,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:01

#进入VM0
service NetworkManager stop
ifconfig eth0 192.168.5.1/24 up
tcpdump -i eth0

#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 （可以ping通）
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 （可以ping通）

# vhost user 启动 VM3
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \
-object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm3.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:30 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap33,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38 \
-chardev socket,id=char1,path=/tmp/sock2,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:03

# 进入VM3
# service NetworkManager stop
ifconfig eth0 192.168.5.2/24 up
ping 192.168.5.1

#进入VM进行配置
#ifconfig eth0 192.168.3.1/24 up
#ping 192.168.3.2 （可以ping通）
#
#ifconfig eth0 192.168.4.1/24 up
#ping 192.168.4.2 （可以ping通）

#删除VM网络
tunctl -d tap30

# 解开内核SRC RPM包
rpm2cpio kernel-3.10.0-1127.el7.src.rpm | cpio -idmv

# 编译 example vhost
cd /home/liufeng/workspace/src/ovs-dpdk/dpdk/examples/vhost
#make RTE_SDK=`pwd`/../.. RTE_TARGET=x86_64-native-linuxapp-gcc
#make RTE_SDK=`pwd`/../.. RTE_TARGET=x86_64-native-linuxapp-gcc V=1
export RTE_TARGET=x86_64-native-linuxapp-gcc
export RTE_SDK=`pwd`/../..
rm -rf ./build
make
#make V=1

################################################################################
# vhost switch 安装
################################################################################

cd /home/liufeng/workspace/src/ovs-dpdk/dpdk/examples/vhost
make RTE_SDK=`pwd`/../.. RTE_TARGET=x86_64-native-linuxapp-gcc

# 启动 vhost-switch
su
sudo ./build/vhost-switch -l 0-3 -n 4 --socket-mem 1024 -- -p 0x1 --socket-file /tmp/sock0 --client

# 启动虚拟机
qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 3072M \
-object memory-backend-file,id=mem,size=3072M,mem-path=/dev/hugepages,share=on \
-hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem \
-vnc 0.0.0.0:00 -monitor stdio --enable-kvm \
-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 \
-chardev socket,id=char1,path=/tmp/sock0,server \
-netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:01

#进入虚拟机操作
su

grep HugePages_ /proc/meminfo

sudo modprobe vfio
sudo modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*

modprobe uio_pci_generic

dpdk-devbind --status
sudo ifconfig eth0 down
dpdk-devbind --bind=uio_pci_generic eth0
dpdk-devbind --status

#testpmd -l 0-1 -d /home/liufeng/workspace/src/dpdk/sdk/lib/librte_pmd_virtio.so -- -i

testpmd -l 0-1 -- -i

show port stats all
start tx_first
show port stats all

leoufung

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
KVM环境初搭建【工作笔记】

######################################################################################安装系统###################################################################################### 按F2进入BIOS# 在CPU 配置中的socket1和socket0中查看是否激活了Inter vt-d的功能# 网络配置接口 ens117f3
复制链接

扫一扫