Mellanox Infiniband卡切换IB/Ethernet模式

最新推荐文章于 2024-09-22 01:03:30 发布

晦暗留给过往

最新推荐文章于 2024-09-22 01:03:30 发布

阅读量2.8w

点赞数 14

分类专栏：网卡 Linux 文章标签：运维服务器

如有转发，请附上链接，谢谢

本文链接：https://blog.csdn.net/qq_44777969/article/details/108033334

版权

Linux 同时被 2 个专栏收录

26 篇文章 0 订阅

订阅专栏

网卡

8 篇文章 1 订阅

订阅专栏

在RedHat或者CentOS系统下切换 Infiniband卡的工作模式

Infiniband卡支持两种工作模式：IB模式和 Ethernet模式

首先，启动mst 工具，通过 mst工具查看自己的MST devices：/dev/mst/mt4119_pciconf0 （没有mst工具，需要下载安装）

[root@localhost ~]# systemctl start mst
[root@localhost ~]# mst status
MST modules:
------------
    MST PCI module is not loaded
    MST PCI configuration module loaded

MST devices:
------------
/dev/mst/mt4119_pciconf0         - PCI configuration cycles access.
                                   domain:bus:dev.fn=0000:c4:00.0 addr.reg=88 data.reg=92 cr_bar.gw_offset=-1
                                   Chip revision is: 00
[root@localhost ~]#

查看网卡的配置信息，在下列输出中，可以看出此时网卡的工作模式是IB模式
也可以通过ifconfig去查看

[root@localhost ~]# mlxconfig -d /dev/mst/mt4119_pciconf0 query

Device #1:
----------

Device type:    ConnectX5       
Name:           MCX555A-ECA_Ax  
Description:    ConnectX-5 VPI adapter card; EDR IB (100Gb/s) and 100GbE; single-port QSFP28; PCIe3.0 x16; tall bracket; ROHS R6
Device:         /dev/mst/mt4119_pciconf0

Configurations:                              Next Boot
         MEMIC_BAR_SIZE                      0               
         MEMIC_SIZE_LIMIT                    _256KB(1)       
         HOST_CHAINING_MODE                  DISABLED(0)     
         HOST_CHAINING_DESCRIPTORS           Array[0..7]     
         HOST_CHAINING_TOTAL_BUFFER_SIZE     Array[0..7]     
         FLEX_PARSER_PROFILE_ENABLE          0               
         FLEX_IPV4_OVER_VXLAN_PORT           0               
         ROCE_NEXT_PROTOCOL                  254             
         ESWITCH_HAIRPIN_DESCRIPTORS         Array[0..7]     
         ESWITCH_HAIRPIN_TOT_BUFFER_SIZE     Array[0..7]     
         PF_BAR2_SIZE                        0               
         NON_PREFETCHABLE_PF_BAR             False(0)        
         VF_VPD_ENABLE                       False(0)        
         STRICT_VF_MSIX_NUM                  False(0)        
         VF_NODNIC_ENABLE                    False(0)        
         NUM_OF_VFS                          0               
         PF_BAR2_ENABLE                      False(0)        
         SRIOV_EN                            False(0)        
         PF_LOG_BAR_SIZE                     5               
         VF_LOG_BAR_SIZE                     1               
         NUM_PF_MSIX                         63              
         NUM_VF_MSIX                         19              
         INT_LOG_MAX_PAYLOAD_SIZE            AUTOMATIC(0)    
         PARTIAL_RESET_EN                    False(0)        
         SW_RECOVERY_ON_ERRORS               False(0)        
         RESET_WITH_HOST_ON_ERRORS           False(0)        
         ADVANCED_POWER_SETTINGS             False(0)        
         CQE_COMPRESSION                     BALANCED(0)     
         IP_OVER_VXLAN_EN                    False(0)        
         MKEY_BY_NAME                        False(0)        
         ESWITCH_IPV4_TTL_MODIFY_ENABLE      False(0)        
         PRIO_TAG_REQUIRED_EN                False(0)        
         UCTX_EN                             True(1)         
         PCI_ATOMIC_MODE                     PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)
         TUNNEL_ECN_COPY_DISABLE             False(0)        
         LRO_LOG_TIMEOUT0                    6               
         LRO_LOG_TIMEOUT1                    7               
         LRO_LOG_TIMEOUT2                    8               
         LRO_LOG_TIMEOUT3                    13              
         LOG_DCR_HASH_TABLE_SIZE             11              
         DCR_LIFO_SIZE                       16384           
         LINK_TYPE_P1                        IB(1)    #当前工作模式 IB      
         ROCE_CC_PRIO_MASK_P1                255             
         ROCE_CC_ALGORITHM_P1                ECN(0)          
         CLAMP_TGT_RATE_AFTER_TIME_INC_P1    True(1)         
         CLAMP_TGT_RATE_P1                   False(0)        
         RPG_TIME_RESET_P1                   300             
         RPG_BYTE_RESET_P1                   32767           
         RPG_THRESHOLD_P1                    1               
         RPG_MAX_RATE_P1                     0               
         RPG_AI_RATE_P1                      5               
         RPG_HAI_RATE_P1                     50              
         RPG_GD_P1                           11              
         RPG_MIN_DEC_FAC_P1                  50              
         RPG_MIN_RATE_P1                     1               
         RATE_TO_SET_ON_FIRST_CNP_P1         0               
         DCE_TCP_G_P1                        1019            
         DCE_TCP_RTT_P1                      1               
         RATE_REDUCE_MONITOR_PERIOD_P1       4               
         INITIAL_ALPHA_VALUE_P1              1023            
         MIN_TIME_BETWEEN_CNPS_P1            2               
         CNP_802P_PRIO_P1                    6               
         CNP_DSCP_P1                         48              
         LLDP_NB_DCBX_P1                     False(0)        
         LLDP_NB_RX_MODE_P1                  OFF(0)          
         LLDP_NB_TX_MODE_P1                  OFF(0)          
         DCBX_IEEE_P1                        True(1)         
         DCBX_CEE_P1                         True(1)         
         DCBX_WILLING_P1                     True(1)         
         KEEP_ETH_LINK_UP_P1                 True(1)         
         KEEP_IB_LINK_UP_P1                  False(0)        
         KEEP_LINK_UP_ON_BOOT_P1             False(0)        
         KEEP_LINK_UP_ON_STANDBY_P1          False(0)        
         NUM_OF_VL_P1                        _4_VLs(3)       
         NUM_OF_TC_P1                        _8_TCs(0)       
         NUM_OF_PFC_P1                       8               
         DUP_MAC_ACTION_P1                   LAST_CFG(0)     
         SRIOV_IB_ROUTING_MODE_P1            LID(1)          
         IB_ROUTING_MODE_P1                  LID(1)          
         PCI_WR_ORDERING                     per_mkey(0)     
         MULTI_PORT_VHCA_EN                  False(0)        
         PORT_OWNER                          True(1)         
         ALLOW_RD_COUNTERS                   True(1)         
         RENEG_ON_CHANGE                     True(1)         
         TRACER_ENABLE                       True(1)         
         IP_VER                              IPv4(0)         
         BOOT_UNDI_NETWORK_WAIT              0               
         UEFI_HII_EN                         True(1)         
         BOOT_DBG_LOG                        False(0)        
         UEFI_LOGS                           DISABLED(0)     
         BOOT_VLAN                           1               
         LEGACY_BOOT_PROTOCOL                PXE(1)          
         BOOT_RETRY_CNT                      NONE(0)         
         BOOT_INTERRUPT_DIS                  False(0)        
         BOOT_LACP_DIS                       True(1)         
         BOOT_VLAN_EN                        False(0)        
         BOOT_PKEY                           0               
         ATS_ENABLED                         False(0)        
         DYNAMIC_VF_MSIX_TABLE               False(0)        
         EXP_ROM_UEFI_x86_ENABLE             True(1)         
         EXP_ROM_PXE_ENABLE                  True(1)         
         IBM_TUNNELED_ATOMIC_EN              False(0)        
         IBM_AS_NOTIFY_EN                    False(0)        
         ADVANCED_PCI_SETTINGS               False(0)        
         SAFE_MODE_THRESHOLD                 10              
         SAFE_MODE_ENABLE                    True(1)         
[root@localhost ~]#

修改网卡的工作模式：
Ethernet模式： mlxconfig -d /dev/mst/mt4119_pciconf0 set LINK_TYPE_P1=2
IB模式： mlxconfig -d /dev/mst/mt4119_pciconf0 set LINK_TYPE_P1=1

[root@localhost ~]# mlxconfig -d /dev/mst/mt4119_pciconf0 set LINK_TYPE_P1=2

Device #1:
----------

Device type:    ConnectX5       
Name:           MCX555A-ECA_Ax  
Description:    ConnectX-5 VPI adapter card; EDR IB (100Gb/s) and 100GbE; single-port QSFP28; PCIe3.0 x16; tall bracket; ROHS R6
Device:         /dev/mst/mt4119_pciconf0

Configurations:                              Next Boot       New
         LINK_TYPE_P1                        IB(1)           ETH(2)          

 Apply new Configuration? (y/n) [n] : y
Applying... Done!
-I- Please reboot machine to load new configurations.
[root@localhost ~]#