1、试验说明
硬件设备:两台鲲鹏920服务器
软件说明:统信操作系统,内核版本4.19.90-2305.1.0.0199.56.9.uel20.aarch64
2、网卡说明
当前的鲲鹏920服务器配置为自带4个25G网卡
网卡名称如下:
速率如下:
由于当前光模块只有10Gb,速率link为10G。
其中只有两个网卡支持RDMA
3、RDMA性能测试
send性能测试
root@node1-2 ~]# ib_send_bw -d hns_2 192.168.104.10
---------------------------------------------------------------------------------------
Send BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
TX depth : 128
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x000c PSN 0xb94a45
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
remote address: LID 0000 QPN 0x000c PSN 0xa32129
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
65536 1000 1165.18 1165.17 0.018643
---------------------------------------------------------------------------------------
[root@localhost ~]# ib_send_bw -d hns_2
************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
Send BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
RX depth : 512
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x000c PSN 0xa32129
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
remote address: LID 0000 QPN 0x000c PSN 0xb94a45
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
65536 1000 0.00 1166.48 0.018664
---------------------------------------------------------------------------------------
read带宽性能测试
[root@localhost Desktop]# ib_read_bw -d hns_2 192.168.104.20
---------------------------------------------------------------------------------------
Device not recognized to implement inline feature. Disabling it
---------------------------------------------------------------------------------------
RDMA_Read BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
TX depth : 128
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Outstand reads : 128
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x0009 PSN 0x56ace7 OUT 0x80 RKey 0x000300 VAddr 0x00ffff8ebb2000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
remote address: LID 0000 QPN 0x0009 PSN 0xac9cde OUT 0x80 RKey 0x000300 VAddr 0x00ffffa3ef7000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
65536 1000 1165.18 1165.13 0.018642
---------------------------------------------------------------------------------------
[root@node1-2 ~]# ib_read_bw -d hns_2
---------------------------------------------------------------------------------------
Device not recognized to implement inline feature. Disabling it
************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
RDMA_Read BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Outstand reads : 128
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x0009 PSN 0xac9cde OUT 0x80 RKey 0x000300 VAddr 0x00ffffa3ef7000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
remote address: LID 0000 QPN 0x0009 PSN 0x56ace7 OUT 0x80 RKey 0x000300 VAddr 0x00ffff8ebb2000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
65536 1000 1165.18 1165.13 0.018642
---------------------------------------------------------------------------------------
write带宽性能测试
[root@localhost Desktop]# ib_write_bw -d hns_2 192.168.104.20
---------------------------------------------------------------------------------------
RDMA_Write BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
TX depth : 128
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x000a PSN 0xb70bcb RKey 0x000300 VAddr 0x00ffff932f9000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
remote address: LID 0000 QPN 0x000a PSN 0x2489a4 RKey 0x000300 VAddr 0x00ffffa30fa000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
65536 5000 1165.18 1165.06 0.018641
---------------------------------------------------------------------------------------
[root@node1-2 ~]# ib_write_bw -d hns_2
************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
RDMA_Write BW Test
Dual-port : OFF Device : hns_2
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
CQ Moderation : 100
Mtu : 4096[B]
Link type : Ethernet
GID index : 2
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0000 QPN 0x000a PSN 0x2489a4 RKey 0x000300 VAddr 0x00ffffa30fa000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:20
remote address: LID 0000 QPN 0x000a PSN 0xb70bcb RKey 0x000300 VAddr 0x00ffff932f9000
GID: 00:00:00:00:00:00:00:00:00:00:255:255:192:168:104:10
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps]
Your system is not activated. Please activate as soon as possible for normal use.
65536 5000 1165.18 1165.06 0.018641
---------------------------------------------------------------------------------------