DPDK-VPP 学习笔记-01

install with CentOS yum

yum update -y 
yum -y install epel-release dpdk-tools
curl -s https://packagecloud.io/install/repositories/fdio/release/script.rpm.sh | sudo bash
yum -y install vpp vpp-plugins vpp-devel vpp-api-python vpp-api-lua

vpp-debuginfo vpp-devel 可选择性安装.

install with source code

注:

建议使用`CentOS7`系统, 避免安装依赖工具失败; 
建议安装 devtoolset-9, 使用 gcc9 作为编译工具;

环境准备

源代码可以直接通过git clone下载到本地, 也可以选择下载指定版本源代码. 这里选择的git clone 方式 & vpp版本是v20.05.1,:

yum update
git clone https://gerrit.fd.io/r/vpp
cd vpp
git checkout v20.05.1 -b 20.05.1
make install-dep

make install-dep会自动下载相应的dpdk版本&依赖库.

编译vpp

注: 下载源代码解压无法编译rpm & deb安装包.

make debug
make debug-release
make build
make build-release
make rebuild
make rebuild-release
make pkg-rpm

清理 vpp

make wipe
make wipe-release

编译参数

新版vpp默认是不支持 dpdk 的, 而 dpdk pmd mlx5 也是默认不支持, 都需要手动指定编译参数.
具体可以查看 build-data/platforms/vpp.mk 与 build/external/packages/dpdk.mk Makefile源代码, 以下列出几个比较重要的编译参数:

参数	描叙
vpp_uses_dpdk=yes	启用 vpp dpdk
DPDK_MLX5_PMD=y	启用 dpdk pmd mlx5
DPDK_MLX5_PMD_DLOPEN_DEPS=y
vpp_uses_dpdk_mlx5_pmd=yes
DPDK_MACHINE=corei7-avx	DPDK CPU平台优化参数, 具体man gcc 查看-mtune/march
DPDK_TUNE=corei7-avx

编译示例:

make \
	vpp_uses_dpdk=yes \
	DPDK_MLX5_PMD=y  DPDK_MLX5_PMD_DLOPEN_DEPS=y  \
	DPDK_MACHINE=corei7-avx DPDK_TUNE=corei7-avx \
	vpp_uses_dpdk_mlx5_pmd=yes \
	vpp_configure_args_vpp="--enable-dlmalloc" \
	build-release

编译后在 vpp 目录下生成 dpdk & vpp 相应目录:

build-root/install-vpp-native/external
build-root/install-vpp-native/vpp

注:

vpp 不支持像dpdk一样使用 EXTRA_CFLAGS & EXTRA_LDFLAGS 作为附带编译&链接参数, 如需支持这类编译方式, 需自己手动修改 build/external/packages/dpdk.mk 文件中 DPDK_EXTRA_CFLAGS :
diff --git a/build/external/packages/dpdk.mk b/build/external/packages/dpdk.mk
index c4e6dc7..366cce2 100644
--- a/build/external/packages/dpdk.mk
+++ b/build/external/packages/dpdk.mk
@@ -135,9 +135,9 @@ DPDK_CPU_CFLAGS := -pie -fPIC
 endif
 
 ifeq ($(DPDK_DEBUG),n)
-DPDK_EXTRA_CFLAGS := -g -mtune=$(DPDK_TUNE)
+DPDK_EXTRA_CFLAGS += -g -mtune=$(DPDK_TUNE)
 else
-DPDK_EXTRA_CFLAGS := -g -O0
+DPDK_EXTRA_CFLAGS += -g -O0
 endif
 
如需修改默认 dpdk config build/external/packages/dpdk.mk 以下段落, 添加相应dpdk配置参数:
...
$(call set,RTE_PKTMBUF_HEADROOM,$(DPDK_PKTMBUF_HEADROOM))
...

vpp 使用

确认系统网卡型号:

#  lshw -businfo -c network
Bus info          Device     Class          Description
=======================================================
pci@0000:02:00.0  em1        network        I350 Gigabit Network Connection
pci@0000:02:00.3  em2        network        I350 Gigabit Network Connection
pci@0000:05:00.0  em3        network        82599ES 10-Gigabit SFI/SFP+ Network Connection
pci@0000:05:00.1  em4        network        82599ES 10-Gigabit SFI/SFP+ Network Connection

重新绑定网卡到 igb_uio 驱动:

# insmod igb_uio.ko
# dpdk-devbind.py -b igb_uio 05:00.0
# dpdk-devbind.py -s

Network devices using DPDK-compatible driver
============================================
0000:05:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' drv=igb_uio unused=ixgbe

Network devices using kernel driver
===================================
0000:02:00.0 'I350 Gigabit Network Connection 1521' if=em1 drv=igb unused=igb_uio *Active*
0000:02:00.3 'I350 Gigabit Network Connection 1521' if=em2 drv=igb unused=igb_uio *Active*
0000:05:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=em4 drv=ixgbe unused=igb_uio *Active*
===================================

初始化 hugepages 大小, 这里使用默认 2M page 大小分配 2048 * 2 * 2 = 8192M

# for n in /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages; do echo 2048 > $n; done
# cat /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
128
128

startup.conf

unix {
  nodaemon
  log /var/log/vpp/vpp.log
  full-coredump
  cli-listen /run/vpp/cli.sock
  gid vpp
  startup-config /root/vpp.cfg # 指定启动加载 vpp 配置文件
}
api-trace {
  on
}
api-segment {
  gid vpp
}
socksvr {
  default
}
# 分配指定 CPU 核
cpu {
	main-core 0
	corelist-workers 1-8,16-24
}
# 
dpdk {
    # 配置dpdk port 默认参数
    # 配置 devargs 可指定pmd默认参数
	dev default {
		num-rx-queues 4
		num-tx-queues 4
		num-rx-desc 512
		num-tx-desc 512
	}
	# 配置 dpdk 绑定 device
	dev 0000:05:00.0
	no-multi-seg
	# 配置 dpdk memory
	socket-mem 2048,2048
}
plugins {
    # 配置 plugins 路径
	path /mnt/source/dpdk-vpp-snb/build-root/install-vpp-native/vpp/lib/vpp_plugins
	# 配置 plugins 默认状态 & 单个plugin状态.
	# plugin default { disable }
	# plugin dpdk_plugin.so { enable }
	# plugin lb_plugin.so { disable }
	plugin default { enable }
}

启动 vpp

# vpp -c path/startup.conf
vpp[24738]: perfmon: No table for cpuid 206d7
vpp[24738]: perfmon:   model 2d, stepping 7
vpp[24738]: tls_init_ca_chain:609: Could not initialize TLS CA certificates
vpp[24738]: tls_mbedtls_init:644: failed to initialize TLS CA chain
vpp[24738]: tls_init_ca_chain:874: Could not initialize TLS CA certificates
vpp[24738]: tls_openssl_init:948: failed to initialize TLS CA chain
vpp[24738]: clib_sysfs_read: open `/sys/devices/system/node/node1/hugepages/hugepages-2048kB/free_hugepages': No such file or directory
vpp[24738]: dpdk_config:1455: socket-mem argument is deprecated
vpp[24738]: dpdk: EAL init args: -c 1ff01ff -n 4 --in-memory --file-prefix vpp -w 0000:05:00.0 --master-lcore 0 

vppctl

# vppctl
    _______    _        _   _____  ___
 __/ __/ _ \  (_)__    | | / / _ \/ _ \
 _/ _// // / / / _ \   | |/ / ___/ ___/
 /_/ /____(_)_/\___/   |___/_/  /_/

vpp# 

vppctl 常用命令(XXXX表示接口名称-GigabitEthernet5/0/0 or TenGigabitEthernet5/0/0 … ):

show pci                 # 查看系统PCI设备
show hardware-interfaces # 查看网络接口硬件
show interface           # 查看网络接口状态
show interface addr      # 查看网络接口地址
show ip fib              # 查看ipv4路由表
show ipv6 fib            # 查看ipv6路由表
set int packet/l3/ip4/ip6/mpls mtu size XXXX              # 配置网络接口MTU, 默认是9000
set int state XXXX up    # up 网络接口
set int ip address  XXXX X.X.X.X/X # 配置IPv4地址
ip route add X.X.X.X/X via X.X.X.X XXXX
ip route del X.X.X.X/X via X.X.X.X XXXX

vpp# show pci
Address      Sock VID:PID     Link Speed   Driver          Product Name                    Vital Product Data
0000:02:00.0   0  8086:1521   5.0 GT/s x4  igb                                             
0000:02:00.3   0  8086:1521   5.0 GT/s x4  igb                                             
0000:05:00.0   0  8086:10fb   5.0 GT/s x8  igb_uio                                         
0000:05:00.1   0  8086:10fb   5.0 GT/s x8  ixgbe                                           
vpp# show hardware-interfaces 
              Name                Idx   Link  Hardware
TenGigabitEthernet5/0/0            1    down  TenGigabitEthernet5/0/0
  Link speed: unknown
  Ethernet address 90:e2:ba:0a:73:0c
  Intel 82599
    carrier down 
    flags: pmd tx-offload intel-phdr-cksum rx-ip4-cksum
    Devargs: 
    rx: queues 4 (max 128), desc 512 (min 32 max 4096 align 8)
    tx: queues 4 (max 64), desc 512 (min 32 max 4096 align 8)
    pci: device 8086:10fb subsystem 8086:7a11 address 0000:05:00.00 numa 0
    max rx packet len: 15872
    promiscuous: unicast off all-multicast off
    vlan offload: strip off filter off qinq off
    rx offload avail:  vlan-strip ipv4-cksum udp-cksum tcp-cksum tcp-lro 
                       macsec-strip vlan-filter vlan-extend jumbo-frame scatter 
                       security keep-crc rss-hash 
    rx offload active: ipv4-cksum 
    tx offload avail:  vlan-insert ipv4-cksum udp-cksum tcp-cksum sctp-cksum 
                       tcp-tso macsec-insert multi-segs security 
    tx offload active: udp-cksum tcp-cksum 
    rss avail:         ipv4-tcp ipv4-udp ipv4 ipv6-tcp-ex ipv6-udp-ex ipv6-tcp 
                       ipv6-udp ipv6-ex ipv6 
    rss active:        none
    tx burst function: ixgbe_xmit_pkts
    rx burst function: ixgbe_recv_pkts

local0                             0    down  local0
  Link speed: unknown
  local
vpp# show  interface 
              Name               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count     
TenGigabitEthernet5/0/0           1     down         9000/0/0/0     
local0                            0     down          0/0/0/0         
vpp# show interface addr
TenGigabitEthernet5/0/0 (dn):
local0 (dn):

vpp.cfg

set int mtu packet 1500 TenGigabitEthernet5/0/0
set int ip address  TenGigabitEthernet5/0/0 40.0.148.68/24
set int state  TenGigabitEthernet5/0/0 up
show ip fib

参考:

FD.io VPP v20.05.1