简析vpp中的dpdk初始化

最近想手撸dpdk，正好需要研究一下dpdk的参数如何配置，就从vpp出发看看dpdk如何初始化的。

vpp使用默认startup.conf配置。

全局初始化rte_eal_init

跟踪到执行该函数的参数

[root@localhost fdio-vpp]# make debug
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-119.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /root/ws/fdio-vpp/build-root/install-vpp_debug-native/vpp/bin/vpp...done.
Signal        Stop      Print   Pass to program Description
SIGUSR1       No        No      Yes             User defined signal 1
(gdb) b init.c:1495
No source file named init.c.
Make breakpoint pending on future shared library load? (y or [n]) y
Breakpoint 1 (init.c:1495) pending.
(gdb) run
Starting program: /root/ws/fdio-vpp/build-root/install-vpp_debug-native/vpp/bin/vpp unix \{ interactive log /var/log/vpp/vpp.log full-coredump cli-listen /run/vpp/cli.sock gid vpp \} api-trace \{ on \} api-segment \{ gid vpp \} socksvr \{ default \} cpu \{ \}
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
unix_config:471: couldn't open log '/var/log/vpp/vpp.log'
tls_init_ca_chain:609: Could not initialize TLS CA certificates
tls_mbedtls_init:644: failed to initialize TLS CA chain
tls_init_ca_chain:874: Could not initialize TLS CA certificates
tls_openssl_init:948: failed to initialize TLS CA chain
vlib_pci_bind_to_uio: Skipping PCI device 0000:03:00.0 as host interface ens160 is up

Breakpoint 1, dpdk_config (vm=0x7ffff7f8b780 <vlib_global_main>, input=0x7fffb27a1788 <_vlib_config_function_dpdk_config+8>)
    at /root/ws/fdio-vpp/src/plugins/dpdk/device/init.c:1495
1495      ret = rte_eal_init (vec_len (conf->eal_init_args),
Missing separate debuginfos, use: debuginfo-install keyutils-libs-1.5.8-3.el7.x86_64 libgcc-4.8.5-39.el7.x86_64 libselinux-2.5-15.el7.x86_64 libuuid-2.23.2-63.el7.x86_64 mbedtls-2.7.17-1.el7.x86_64 numactl-libs-2.0.12-5.el7.x86_64 pcre-8.32-17.el7.x86_64 pkcs11-helper-1.11-3.el7.x86_64
(gdb) p conf->eal_init_args
$1 = (u8 **) 0x7fffb5e90cd0
(gdb) p conf->eal_init_args_str
$2 = (u8 *) 0x7fffb5ff7610 "-c 2 -n 4 --in-memory --file-prefix vpp -b 0000:03:00.0 --master-lcore 1 "

-c 2 -n 4 --in-memory --file-prefix vpp -b 0000:03:00.0 --master-lcore 1

-c 2  :  逻辑核心掩码 binary: 10  ，即第二颗logic core

-n 4 : 内存通道数 默认初始化成4， 一般家用PC只有2通道，服务器才有4通道

--in-memory: 全内存中运行，不创建共享数据结构

--file-prefix: 大页内存映射的文件名前缀

-b 0000:03:00.0: 不enable 这个接口，-b代表blacklist, 因为此接口已经由系统接管而且使用了。

--master-lcore 1: 主核心为 1 （从0 开始数）

top - 02:04:00 up 13:08,  3 users,  load average: 1.00, 0.99, 0.85
Tasks: 214 total,   2 running, 212 sleeping,   0 stopped,   0 zombie
%Cpu0  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu1  :100.0 us,  0.0 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu2  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu3  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu4  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu5  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu6  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu7  :  0.3 us,  0.0 sy,  0.0 ni, 99.7 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  7990076 total,   934604 free,  5060988 used,  1994484 buff/cache
KiB Swap:  4064252 total,  4045820 free,    18432 used.  2616140 avail Mem 

网卡初始化dpdk_device_setup

RTE_ETH_FOREACH_DEV(i) 遍历每一个网卡

每个网卡配置相关参数

  rv = rte_eth_dev_configure (xd->port_id, xd->rx_q_used,
			      xd->tx_q_used, &xd->port_conf);
// 设置该网卡rx队列和tx队列数量
// port_id为该网卡id

// 每一个队列，都设置一个队列大小，默认是1024.
// 队列数量和队列大小会影响到l3-cache的命中。
 
 /* Set up one TX-queue per worker thread */
  for (j = 0; j < xd->tx_q_used; j++)
    {
      rv =
	rte_eth_tx_queue_setup (xd->port_id, j, xd->nb_tx_desc,
				xd->cpu_socket, &xd->tx_conf);

      /* retry with any other CPU socket */
      if (rv < 0)
	rv =
	  rte_eth_tx_queue_setup (xd->port_id, j,
				  xd->nb_tx_desc, SOCKET_ID_ANY,
				  &xd->tx_conf);
      if (rv < 0)
	dpdk_device_error (xd, "rte_eth_tx_queue_setup", rv);
    }

  for (j = 0; j < xd->rx_q_used; j++)
    {
      uword tidx = vnet_get_device_input_thread_index (dm->vnet_main,
						       xd->hw_if_index, j);
      unsigned lcore = vlib_worker_threads[tidx].cpu_id;
      u16 socket_id = rte_lcore_to_socket_id (lcore);
      u8 bpidx = vlib_buffer_pool_get_default_for_numa (vm, socket_id);
      vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, bpidx);
      struct rte_mempool *mp = dpdk_mempool_by_buffer_pool_index[bpidx];

      rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->nb_rx_desc,
				   xd->cpu_socket, 0, mp);

      /* retry with any other CPU socket */
      if (rv < 0)
	rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->nb_rx_desc,
				     SOCKET_ID_ANY, 0, mp);

      xd->buffer_pool_for_queue[j] = bp->index;

      if (rv < 0)
	dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv);
    }