NVIDIA BF2网卡进行网络隔离及ib_traffic压测

NVIDIA BF3网卡进行网络隔离及ib_traffic压测

提示：这里简述项目相关背景：同机台进行IB打流时候，可以进行网络隔离，设置每张网卡网口的单独网络环境，进行流量转发操作；

例如：BF3调试及工具安装跳过；主要介绍网络隔离IP设置

---

#如四张BF3单口400G网卡对联，进行网络IP隔离设置
`提示：这里描述项目中遇到的问题：单机台网卡互联默认无法进行流量转发；故借用防火墙规则进行网络IP隔离

例如：网卡信息如下

增加网络隔离：

 sudo ip rule show
  sudo ip rule add from all lookup local pref 100
  sudo ip rule show
  sudo ip rule del pref 0
  cat /proc/sys/net/ipv4/conf/all/rp_filter
  sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter'

  ibdev2netdev | grep enp28s0f0np0        #NIC1
  cat /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter
  cat /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local
  cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter
  cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore
  cat /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce
  sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore'
  sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce'
  sudo ifconfig enp28s0f0np0 192.168.103.1/24 up
  sudo ip rule add iif enp28s0f0np0 lookup local pref 0
  sudo ip rule add from 192.168.103.1 table 10 pref 10
  sudo ip route add 192.168.103.0/24 dev enp28s0f0np0 src 192.168.103.1 table 10
  sudo ip route add local 192.168.103.1 dev enp28s0f0np0 src 192.168.103.1 table 10
  sudo ip rule add to 192.168.103.1 table 10 pref 10
  ibdev2netdev | grep enp28s0f0np0

  ibdev2netdev | grep enp58s0f0np0            #NIC2
  cat /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter
  cat /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local
  cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter
  cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore
  cat /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce
  sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore'
  sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce'
  sudo ifconfig enp58s0f0np0 192.168.103.2/24 up
  sudo ip rule add iif enp58s0f0np0 lookup local pref 0
  sudo ip rule add from 192.168.103.2 table 11 pref 10
  sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.2 table 11
  sudo ip route add local 192.168.103.2 dev enp58s0f0np0 src 192.168.103.2 table 11
  sudo ip rule add to 192.168.103.2 table 11 pref 10
  ibdev2netdev | grep enp58s0f0np0

ibdev2netdev | grep enp158s0f0np0            #NIC3
  cat /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter
  cat /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local
  cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter
  cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore
  cat /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce
  sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore'
  sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce'
  sudo ifconfig enp158s0f0np0 192.168.103.3/24 up
  sudo ip rule add iif enp158s0f0np0 lookup local pref 0
  sudo ip rule add from 192.168.103.3 table 12 pref 10
  sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.3 table 12
  sudo ip route add local 192.168.103.3 dev enp58s0f0np0 src 192.168.103.3 table 12
  sudo ip rule add to 192.168.103.3 table 12 pref 10
  ibdev2netdev | grep enp158s0f0np0

ibdev2netdev | grep enp186s0f0np0                               #NIC4
  cat /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter
  cat /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local
  cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter
  cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore
  cat /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce
  sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter'
  sudo bash -c 'echo 1 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore'
  sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce'
  sudo ifconfig enp186s0f0np0 192.168.103.4/24 up
  sudo ip rule add iif enp186s0f0np0 lookup local pref 0
  sudo ip rule add from 192.168.103.4 table 13 pref 10
  sudo ip route add 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.4 table 13
  sudo ip route add local 192.168.103.4 dev enp58s0f0np0 src 192.168.103.4 table 13
  sudo ip rule add to 192.168.103.4 table 13 pref 10
  ibdev2netdev | grep enp58s0f0np0

  sudo ip -s -s neigh flush all
  sudo ip route flush cache

删除网络IP隔离

sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp28s0f0np0/rp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/accept_local'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_ignore'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp28s0f0np0/arp_announce'
   sudo ip rule del to 192.168.103.1 table 10 pref 10               #NIC1
   sudo ip route del local 192.168.103.1 dev enp28s0f0np0 src 192.168.103.1 table 10
   sudo ip route del 192.168.103.0/24 dev enp28s0f0np0 src 192.168.103.1 table 10
   sudo ip rule del from 192.168.103.1 table 10 pref 10
   sudo ip rule del iif enp28s0f0np0 lookup local pref 0


sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp58s0f0np0/rp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/accept_local'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_ignore'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp58s0f0np0/arp_announce'
   sudo ip rule del to 192.168.103.2 table 11 pref 10                 #NIC2
   sudo ip route del local 192.168.103.2 dev enp58s0f0np0 src 192.168.103.2 table 11
   sudo ip route del 192.168.103.0/24 dev enp58s0f0np0 src 192.168.103.2 table 11
   sudo ip rule del from 192.168.103.2 table 11 pref 10
   sudo ip rule del iif enp58s0f0np0 lookup local pref 0
sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp158s0f0np0/rp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/accept_local'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_ignore'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp158s0f0np0/arp_announce'
   sudo ip rule del to 192.168.103.3 table 12 pref 10           #NIC3
   sudo ip route del local 192.168.103.3 dev enp158s0f0np0 src 192.168.103.3 table 12
   sudo ip route del 192.168.103.0/24 dev enp158s0f0np0 src 192.168.103.3 table 12
   sudo ip rule del from 192.168.103.3 table 12 pref 10
   sudo ip rule del iif enp158s0f0np0 lookup local pref 0

sudo bash -c 'echo 2 > /proc/sys/net/ipv4/conf/enp186s0f0np0/rp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/accept_local'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_filter'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_ignore'
   sudo bash -c 'echo 0 > /proc/sys/net/ipv4/conf/enp186s0f0np0/arp_announce'
   sudo ip rule del to 192.168.103.4 table 13 pref 10           #NIC4
   sudo ip route del local 192.168.103.4 dev enp186s0f0np0 src 192.168.103.4 table 13
   sudo ip route del 192.168.103.0/24 dev enp186s0f0np0 src 192.168.103.4 table 13
   sudo ip rule del from 192.168.103.4 table 13 pref 10
   sudo ip rule del iif enp186s0f0np0 lookup local pref 0
   
sudo ip rule show
   sudo ip rule add from all lookup local pref 0
   sudo ip rule show
   sudo ip rule del pref 100
   sudo sysctl -w net.ipv4.conf.all.rp_filter=2
   sudo ifconfig enp28s0f0np0 down
   sudo ifconfig enp58s0f0np0 down
   sudo ifconfig enp158s0f0np0 down
     sudo ifconfig enp186s0f0np0 down

设置网络IP后，可以利用iperf或者IB_write_bw进行打流测试，针对test阶段的一些思路，如有问题，希望可以及时交流；

https://blog.csdn.net/liaomin416100569/article/details/126782055 #简单概述通过linux的虚拟网络隔离，来模拟熟悉交换机，路由器，网卡,网桥等物理设备间的关系和区别。

IB_write_bw命令示例：

  ib_write_bw --ib-dev=mlx5_0 --ib-port=1 --port=10001 --connection=RC --qp=1 --run_infinitely &    

  ib_write_bw --ib-dev=mlx5_1 --ib-port=1 --port=10001 --connection=RC --qp=1 --report_gbits --run_infinitely --CPU-freq --post_list=1 192.168.103.1