ovs+dpdk+docker简单实践

环境

Centos 7.6.1810

dpdk    17.11.6 

ovs       2.9.0

pktgen  3.4.4

lua        5.3.4

docker  1.13.1

安装教程https://blog.csdn.net/wh_computers/article/details/99407047

教程参考的https://blog.csdn.net/me_blue/article/details/78589592，但是原教程有部分省略的部分，导致按照步骤无法复现，本文在此基础上进行了完善

实践

在安装好了以上环境后，下面进行收发包测试

1.配置ovs

启动ovs

#init new ovs database
ovsdb-tool create /usr/local/etc/openvswitch/conf.db ./vswitchd/vswitch.ovsschema
#start database server
ovsdb-server --remote=punix:/usr/local/var/run/openvswitch/db.sock --remote=db: Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach --log-file
#initialize ovs database
ovs-vsctl --no-wait init
#configure ovs dpdk 
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true other_config:dpdk-lcore=0x2 other_config:dpdk-socket-mem="1024"
##start ovs
ovs-vswitchd unix:/usr/local/var/run/openvswitch/db.sock --pidfile --detach

创建ovs ports

#ovs use core 2 for the PMD
ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x4
#create br0 and vhost ports which use DPDK，类型dpdkvhostuser表示会创建一个socket文件用来通信，同时这些端口会默认编号1 2 3 4
ovs-vsctl add-br ovs-br0 -- set bridge ovs-br0 datapath_type=netdev
ovs-vsctl add-port ovs-br0 vhost-user0 -- set Interface vhost-user0 type=dpdkvhostuser
ovs-vsctl add-port ovs-br0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuser
ovs-vsctl add-port ovs-br0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuser
ovs-vsctl add-port ovs-br0 vhost-user3 -- set Interface vhost-user3 type=dpdkvhostuser
#show ovs-br0 info
ovs-vsctl show

添加流表

#clear current flows
ovs-ofctl del-flows ovs-br0
#add bi-directional flow between vhost-user1 and vhost-user2(port 2 and 3),2 3 表示端口host1 host2 通信
ovs-ofctl add-flow ovs-br0 in_port=2,dl_type=0x800,idle_timeout=0,action=output:3 
ovs-ofctl add-flow ovs-br0 in_port=3,dl_type=0x800,idle_timeout=0,action=output:2
#add bi-directional flow between vhost-user0 and vhost-user3(port 1 and 4)，1 4表示端口0 3通信
ovs-ofctl add-flow ovs-br0 in_port=1,dl_type=0x800,idle_timeout=0,action=output:4
ovs-ofctl add-flow ovs-br0 in_port=4,dl_type=0x800,idle_timeout=0,action=output:1
#show current flows
ovs-ofctl dump-flows ovs-br0

通过以下命令检查设置

ovs-vsctl show 
ovs-ofctl show ovs-br0
ovs-ofctl dump-flows ovs-br0
ovs-ofctl dump-ports ovs-br0

2.创建容器

创建容器的过程需要联网，如果没有网络需要在有网络的机器上创建好容器拷贝出来

过程是：

docker save docker-name -o docker.tar #把创建的docker-name用docker.tar保存出来，
#拷贝需要权限
chmod 777 docker.tar
#拷贝完执行
docker load -i docker.tar

创建testpmd container 和 l2fwd container

首先make一下l2fwd

cd dpdk-stable-17.11.6/examples/l2fwd
make

接着创建容器

cd dpdk-stable-17.11.6/ #进入dpdk目录下
vim Dockerfile #创建Dockerfile文件
#文件内容是
FROM ubuntu:latest
RUN apt update -y
RUN apt-get install -y  numactl
WORKDIR /root/dpdk
COPY dpdk-stable-17.11.6 /root/dpdk/.
ENV PATH "$PATH:/root/dpdk/x86_64-native-linuxapp-gcc/app/"
#保存文退出 
docker build -t dpdk-docker:17.11 . #通过文件创建docker
docker images #查看docker镜像
[root@localhost wanghe]# docker images
REPOSITORY              TAG                 IMAGE ID            CREATED             SIZE
dpdk-docker             17.11               a3d4aecf55b7        2 hours ago         734 MB

创建pktgen docker

#切换目录
cd pktgen-3.4.4/
vim Dockerfile  #创建文件
#文件内容
FROM ubuntu:latest
RUN apt update -y
RUN apt-get install -y  numactl libpcap-dev
WORKDIR /root/dpdk
COPY dpdk-stable-17.11.6 /root/dpdk/.
COPY pktgen-3.4.4 /root/pktgen/.
RUN ln -s /root/pktgen/app/x86_64-native-linuxapp-gcc/pktgen /usr/bin/pktgen
RUN ln -s /usr/lib/x86_64-linux-gnu/libpcap.so /usr/lib/x86_64-linux-gnu/libpcap.so.1
ENV PATH "$PATH:/root/dpdk/x86_64-native-linuxapp-gcc/app/"
#保存退出
docker build -t pktgen-docker .
docker images #查看docker镜像
[root@localhost wanghe]# docker images
REPOSITORY              TAG                 IMAGE ID            CREATED             SIZE
dpdk-docker             17.11               a3d4aecf55b7        2 hours ago         734 MB
pktgen-docker           latest              5b6592cf8b8e        19 hours ago        746 MB

 3.实验

实验一

启动pktgen

#首先启动docker container
[root@nsfocus]# docker run -ti --privileged --name=pktgen-docker -v /mnt/huge:/mnt/huge -v /usr/local/var/run/openvswitch:/var/run/openvswitch pktgen-docker:latest
#进入以后切换目录
root@0f0617056fe1:~/dpdk# ls
GNUmakefile   Makefile  buildtools  drivers   pkg
LICENSE.GPL   README    config      examples  test
LICENSE.LGPL  app       devtools    lib       usertools
MAINTAINERS   build     doc         mk        x86_64-native-linuxapp-gcc
root@0f0617056fe1:~/dpdk# cd ..
root@0f0617056fe1:~# ls
dpdk  pktgen
root@0f0617056fe1:~# cd pktgen/
root@0f0617056fe1:~/pktgen# 
#切换完成，启动pktgen
root@0f0617056fe1:~/pktgen -c 0x19 --master-lcore 3 -n 1 --socket-mem 1024,1024 --file-prefix pktgen --no-pci  \
--vdev 'net_virtio_user0,mac=00:00:00:00:00:05,path=/var/run/openvswitch/vhost-user0'  \ 
--vdev 'net_virtio_user1,mac=00:00:00:00:00:01,path=/var/run/openvswitch/vhost-user1' \
-- -T -P -m "0.0,4.1"

为了便于观察，这里再开一个终端

启动testpmd

[root@nsfocus ~]# docker run -it --privileged --name=dpdk-docker \ 
-v /mnt/huge:/mnt/huge -v /usr/local/var/run/openvswitch:/var/run/openvswitch \
dpdk-docker:17.11
root@bdbe9fabe89a:~/dpdk/examples/l2fwd/build# cd ~/dpdk/x86_64-native-linuxapp-gcc/app/
root@bdbe9fabe89a:~/dpdk/x86_64-native-linuxapp-gcc/app# 
testpmd -c 0xE0 -n 1 --socket-mem 1024,1024 --file-prefix testpmd --no-pci  \
--vdev 'net_virtio_user2,mac=00:00:00:00:00:02,path=/var/run/openvswitch/vhost-user2' \
--vdev 'net_virtio_user3,mac=00:00:00:00:00:03,path=/var/run/openvswitch/vhost-user3' \
-- -i --burst=64 --disable-hw-vlan --txd=2048 --rxd=2048 -a --coremask=0xc0

发包验证，在pktgen端设置

# 在pktgen中设置速率为10%，更具体的速率设置可以通过tx_cycles设置
# 端口0共发送100个包，端口1发送200个
Pktgen:/>set all rate 10
Pktgen:/>set 0 count 100
Pktgen:/>set 1 count 200
Pktgen:/>str

testpmd端执行

show port stats all

实验二

启动pktgen

#首先启动docker container
#root权限
docker run -ti --privileged --name=pktgen-docker -v /mnt/huge:/mnt/huge -v  /usr/local/var/run/openvswitch:/var/run/openvswitch pktgen-docker:latest
#切换到pktgen目录下
cd ..
cd pktgen
root@a2603b54d66d:~/pktgen# pktgen -c 0x19 --master-lcore 3 -n 1 --socket-mem 1024,1024 --file-prefix pktgen --no-pci --vdev  'net_virtio_user0,mac=00:00:00:00:00:05,path=/var/run/openvswitch/vhost-user0' --vdev  'net_virtio_user1,mac=00:00:00:00:00:01,path=/var/run/openvswitch/vhost-user1' -- -T -P -m "0.0,4.1"

启动l2fwd，前提是容器中已经对examples目录下的l2fwd make过了

#root目录下执行
docker run -it --privileged --name=dpdk-docker  -v /mnt/huge:/mnt/huge -v  /usr/local/var/run/openvswitch:/var/run/openvswitch dpdk-docker:17.11
#切换到examples/l2fwd/build
cd examples/l2fwd/build
root@bdbe9fabe89a:~/dpdk/examples/l2fwd/build#./l2fwd -c 0xE0 -n 1 --socket-mem 1024,1024 --file-prefix testpmd --no-pci --vdev  'net_virtio_user2,mac=00:00:00:00:00:02,path=/var/run/openvswitch/vhost-user2' --vdev 'net_virtio_user3,mac=00:00:00:00:00:03,path=/var/run/openvswitch/vhost-user3' -- -p 0x3

发包验证，在pktgen端输入

# 在pktgen中设置速率为10%，更具体的速率设置可以通过tx_cycles设置
# 端口0共发送100个包，端口1发送200个
Pktgen:/>set all rate 10
Pktgen:/>set 0 count 100
Pktgen:/>set 1 count 200
Pktgen:/>str

通过观察结果证明发包成功