MLNX 网卡 dpdk 适配与dpdk最大队列支持

MLNX 网卡 dpdk 适配

1. 环境

dpdk 版本： dpdk-stable-18.02.1
MLNX 版本： 4.7

2. 下载驱动

https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/

3. 参考文档

https://www.jianshu.com/p/f09f00f4e25f

https://www.jianshu.com/p/638427dd9203

http://doc.dpdk.org/guides-16.11/nics/mlx5.html

http://doc.dpdk.org/guides-20.02/nics/mlx5.html

https://blog.csdn.net/baidu_35141290/article/details/100138087

4. ARM 编译

解压 src MLNX_OFED_SRC…tar.gz 然后移动到MLNX根目录

# 安装Mellanox驱动

yum install gcc gcc-c++ vim wget git net-tools pci*

yum install kernel-devel-4.14.0-115.el7a.0.1.aarch64

yum install python-devel lsof redhat-rpm-config rpm-build

yum install tcl gcc-gfortran tcsh tk

yum -y install libmnl libmnl-devel

yum install mlnx-ofed-all --skip-broken

yum -y install libibverbs*
yum install rdma-core rdma-core-devel -y

# mount -o ro,loop MLNX_OFED_LINUX-4.7-1.0.0.1-rhel7.6alternate-aarch64.iso /mnt/
#./mlnxofedinstall --add-kernel-support --skip-repo --skip-distro-check
#./mlnxofedinstall --skip-distro-check --add-kernel-support --kmp --dpdk --upstream-libs --force --skip-repo
#./mlnxofedinstall --upstream-libs --dpdk --add-kernel-support --force

# 全部安装
#./mlnxofedinstall --all --force

# 支持dpdk的最小化安装
./mlnxofedinstall --dpdk --upstream-libs --force

/etc/init.d/openibd restart

ibstat 命令查看是否安装成功

5. dpdk 测试

modprobe -a ib_uverbs mlx5_core mlx5_ib

/etc/init.d/openibd restart

ifconfig eth1 up
ifconfig eth2 up

ls -d /sys/class/net/*/device/infiniband_verbs/uverbs* | cut -d / -f 5

# eth1 eth2

{
	for intf in eth1 eth2;
	do
		(cd "/sys/class/net/${intf}/device/" && pwd -P);
	done;
} |
sed -n 's,.*/\(.*\),-w \1,p'

# -w 0001:01:00.0 -w 0001:01:00.1

./testpmd -l 8-15 -n 4 -w 0001:01:00.0 -w 0001:01:00.1 -- --rxq=2 --txq=2 -i

6. intel 个网卡支持的队列数

i350 8队列
x710 22队列

igb_ethdev.c
	e1000_82575 4
	e1000_82576 16
	e1000_82580 8
	e1000_i350 8
	e1000_i354 8
	e1000_i210 4
	e1000_i211 2

./net/ixgbe/base/ixgbe_82599.c
	ixgbe 82599 128  光口
./net/ixgbe/base/ixgbe_x540.c
	ixgbe x540 128
./net/ixgbe/base/ixgbe_82598.c
	82598 rx queue 32 / tx queue 64

e1000 (82540, 82545, 82546)
e1000e (82571, 82572, 82573, 82574, 82583, ICH8, ICH9, ICH10, PCH, PCH2, I217, I218, I219)
igb (82573, 82576, 82580, I210, I211, I350, I354, DH89xx)
igc (I225)
ixgbe (82598, 82599, X520, X540, X550)
i40e (X710, XL710, X722, XXV710)
ice (E810)
fm10k (FM10420)
ipn3ke (PAC N3000)
ifc (IFC)

i40e 驱动默认支持 64 队列,见dpdk 网卡最大队列代码追踪.

7. dpdk 网卡最大队列代码追踪

// rte_ethdev.c --> rte_eth_dev_rx_queue_config 设置 nb_queues
static int rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
// 初始dev信息存放在 rte_eth_devices数组中
dev = &rte_eth_devices[port_id];


/*
./lib/librte_ether/rte_ethdev.c:879:	dev->data->nb_tx_queues = nb_queues;

./lib/librte_eventdev/rte_event_eth_rx_adapter.c:907:		rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];
./lib/librte_eventdev/rte_event_eth_rx_adapter.c:1011:			rte_eth_devices[eth_dev_id].data->nb_rx_queues) {
./lib/librte_eventdev/rte_event_eth_rx_adapter.c:1034:				&rte_eth_devices[eth_dev_id],
./lib/librte_eventdev/rte_event_eth_rx_adapter.c:1088:		rte_eth_devices[eth_dev_id].data->nb_rx_queues) {


rte_eth_dev_configure  该函数调用时会修改queue_number个数.
grep -rn "nb_rx_queues" ./
./lib/librte_ether/rte_ethdev.c:1359:	dev->data->nb_rx_queues = 0;



grep -rn "max_rx_queues"

/drivers/net/i40e/i40e_ethdev.c:3173:	dev_info->max_rx_queues = vsi->nb_qps;
*/

static struct rte_event_eth_rx_adapter **event_eth_rx_adapter;
rx_adapter->eth_devices[i].dev = &rte_eth_devices[i];


#define RTE_MAX_QUEUES_PER_PORT 1024 // rte_config.h


int eth_i40e_dev_init(struct rte_eth_dev *dev);
int rte_pci_read_config(const struct rte_pci_device *dev,
	void *buf, size_t len, off_t offset)

// --------------------------------------------------------------------------

// i40e 驱动
// ./drivers/net/i40e/i40e_ethdev.c:3173:	dev_info->max_rx_queues = vsi->nb_qps;            // .dev_infos_get = i40e_dev_info_get

RTE_PMD_REGISTER_PCI(net_i40e, rte_i40e_pmd);
  |
  V
static struct rte_pci_driver rte_i40e_pmd
  |
  V
eth_i40e_pci_probe
  |
  V
eth_i40e_dev_init --------> i40e_pf_parameter_init ---> pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
  |                                                                     pf->lan_nb_qps = pf->lan_nb_qp_max;
  V
i40e_pf_setup                                             
  |
  V
i40e_vsi_setup
  |
  V
vsi->nb_qps = pf->lan_nb_qps