HCA卡的Port GUID 和ib0的关系


HCA卡的Port GUID 和ib0的关系
[root@rac2 ~]# ibstat
CA 'mlx4_0'         ——————————————>>>>额外说明:这是第一个HCA卡(并且是双port,所以下面有port1和prot2)
    CA type: MT4099
    Number of ports: 2
    Firmware version: 2.35.5100
    Hardware version: 0
    Node GUID: 0x248a0703005d07f0
    System image GUID: 0x248a0703005d07f3
    Port 1:
        State: Active
        Physical state: LinkUp
        Rate: 56
        Base lid: 4
        LMC: 0
        SM lid: 4
        Capability mask: 0x0259486a
        Port GUID: 0x248a0703005d07f1---------------->>>注意此处!!!!
        Link layer: InfiniBand
    Port 2:
        State: Down
        Physical state: Polling
        Rate: 10
        Base lid: 0
        LMC: 0
        SM lid: 0
        Capability mask: 0x02594868
        Port GUID: 0x248a0703005d07f2
        Link layer: InfiniBand
CA 'mlx4_1''         ——————————————>>>>额外说明:这是第二个HCA卡(并且是双port,所以下面有port1和prot2)
    CA type: MT4099
    Number of ports: 2
    Firmware version: 2.35.5100
    Hardware version: 0
    Node GUID: 0x248a0703005d0840
    System image GUID: 0x248a0703005d0843
    Port 1:
        State: Active
        Physical state: LinkUp
        Rate: 56
        Base lid: 6
        LMC: 0
        SM lid: 8
        Capability mask: 0x02594868
        Port GUID: 0x248a0703005d0841
        Link layer: InfiniBand
    Port 2:
        State: Down
        Physical state: Polling
        Rate: 10
        Base lid: 0
        LMC: 0
        SM lid: 0
        Capability mask: 0x02594868
        Port GUID: 0x248a0703005d0842
        Link layer: InfiniBand

[root@rac2 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp5s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
    link/ether 0c:c4:7a:ab:d3:4c brd ff:ff:ff:ff:ff:ff
    inet 192.168.100.102/24 brd 192.168.100.255 scope global enp5s0f0
       valid_lft forever preferred_lft forever
    inet6 fe80::ec4:7aff:feab:d34c/64 scope link
       valid_lft forever preferred_lft forever
3: enp5s0f1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 0c:c4:7a:ab:d3:4d brd ff:ff:ff:ff:ff:ff
4: ib0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2044 qdisc pfifo_fast state UP qlen 256
    link/infiniband 80:00:02:08:fe:80:00:00:00:00:00:00:24:8a:07:03:00:5d:07:f1 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
    inet 10.10.10.105/24 brd 10.10.10.255 scope global ib0         ~~~~~~~~~~~~--->注意此处!!!
       valid_lft forever preferred_lft forever
    inet6 fe80::268a:703:5d:7f1/64 scope link
       valid_lft forever preferred_lft forever
5: ib1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 2044 qdisc pfifo_fast state DOWN qlen 256
    link/infiniband 80:00:02:09:fe:80:00:00:00:00:00:00:24:8a:07:03:00:5d:07:f2 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
    inet 10.10.10.106/24 brd 10.10.10.255 scope global ib1
       valid_lft forever preferred_lft forever
6: ib2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2044 qdisc pfifo_fast state UP qlen 256
    link/infiniband 80:00:02:08:fe:80:00:00:00:00:00:02:24:8a:07:03:00:5d:08:41 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
    inet 10.10.10.107/24 brd 10.10.10.255 scope global ib2
       valid_lft forever preferred_lft forever
    inet6 fe80::268a:703:5d:841/64 scope link
       valid_lft forever preferred_lft forever
7: ib3: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 2044 qdisc pfifo_fast state DOWN qlen 256
    link/infiniband 80:00:02:09:fe:80:00:00:00:00:00:00:24:8a:07:03:00:5d:08:42 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
    inet 10.10.10.108/24 brd 10.10.10.255 scope global ib3
       valid_lft forever preferred_lft forever
8: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
    link/ether 52:54:00:74:ff:7b brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
9: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 500
    link/ether 52:54:00:74:ff:7b brd ff:ff:ff:ff:ff:ff


[root@rac2 ~]# ifconfig
enp5s0f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.100.102  netmask 255.255.255.0  broadcast 192.168.100.255
        inet6 fe80::ec4:7aff:feab:d34c  prefixlen 64  scopeid 0x20<link>
        ether 0c:c4:7a:ab:d3:4c  txqueuelen 1000  (Ethernet)
        RX packets 9803  bytes 744701 (727.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 2027  bytes 175450 (171.3 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
        device memory 0xc7820000-c783ffff  

ib0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 2044
        inet 10.10.10.105  netmask 255.255.255.0  broadcast 10.10.10.255
        inet6 fe80::268a:703:5d:7f1  prefixlen 64  scopeid 0x20<link>---------------------------->注意本行最左侧!!
Infiniband hardware address can be incorrect! Please read BUGS section in ifconfig(8).
        infiniband 80:00:02:08:FE:80:00:00:00:00:00:00:00:00:00:00:00:00:00:00  txqueuelen 256  (InfiniBand)
        RX packets 61  bytes 6687 (6.5 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 51  bytes 4298 (4.1 KiB)
        TX errors 0  dropped 7 overruns 0  carrier 0  collisions 0

ib1: flags=4099<UP,BROADCAST,MULTICAST>  mtu 2044
        inet 10.10.10.106  netmask 255.255.255.0  broadcast 10.10.10.255
Infiniband hardware address can be incorrect! Please read BUGS section in ifconfig(8).
        infiniband 80:00:02:09:FE:80:00:00:00:00:00:00:00:00:00:00:00:00:00:00  txqueuelen 256  (InfiniBand)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

ib2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 2044
        inet 10.10.10.107  netmask 255.255.255.0  broadcast 10.10.10.255
        inet6 fe80::268a:703:5d:841  prefixlen 64  scopeid 0x20<link>
Infiniband hardware address can be incorrect! Please read BUGS section in ifconfig(8).
        infiniband 80:00:02:08:FE:80:00:00:00:00:00:00:00:00:00:00:00:00:00:00  txqueuelen 256  (InfiniBand)
        RX packets 14  bytes 3411 (3.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 7 overruns 0  carrier 0  collisions 0

ib3: flags=4099<UP,BROADCAST,MULTICAST>  mtu 2044
        inet 10.10.10.108  netmask 255.255.255.0  broadcast 10.10.10.255
Infiniband hardware address can be incorrect! Please read BUGS section in ifconfig(8).
        infiniband 80:00:02:09:FE:80:00:00:00:00:00:00:00:00:00:00:00:00:00:00  txqueuelen 256  (InfiniBand)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 0  (Local Loopback)
        RX packets 24  bytes 1724 (1.6 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 24  bytes 1724 (1.6 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

virbr0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 192.168.122.1  netmask 255.255.255.0  broadcast 192.168.122.255
        ether 52:54:00:74:ff:7b  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@rac2 ~]# 



  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
RDMA HCA/TCA是一种高速网络适配器,它使用RDMA技术来提高数据传输的效率和性能。HCA代表Host Channel Adapter,而TCA代表Target Channel Adapter。HCA通常安装在主机上,而TCA通常安装在存储设备上。这两种适配器都支持RDMA技术,可以通过RDMA协议进行高速数据传输。 RDMA技术是一种零拷贝技术,它可以直接在内存中传输数据,而不需要将数据从内存复制到网络适配器的缓冲区中。这种技术可以显著提高数据传输的效率和性能,减少CPU的负载,降低网络延迟和网络拥塞。 RDMA HCA/TCA通常使用InfiniBand或者RoCE(RDMA over Converged Ethernet)网络来进行高速数据传输。这些网络可以提供非常低的延迟和高的带宽,适用于高性能计算、云计算、大数据分析等领域。 以下是一个使用RDMA Write with Immediate Data的例子: ```c #include <stdio.h> #include <stdlib.h> #include <string.h> #include <infiniband/verbs.h> #define MSG_SIZE 1024 #define RDMA_BUF_SIZE 1024 struct rdma_context { struct ibv_context *ctx; struct ibv_pd *pd; struct ibv_mr *mr; struct ibv_cq *cq; struct ibv_qp *qp; struct ibv_comp_channel *comp_channel; struct ibv_port_attr port_attr; char *rdma_buf; uint32_t rkey; uint64_t remote_addr; }; int main(int argc, char *argv[]) { struct rdma_context ctx; struct ibv_device **dev_list; struct ibv_device *ib_dev; struct ibv_qp_init_attr qp_init_attr; struct ibv_qp_attr qp_attr; struct ibv_wc wc; int num_devices; int ret; int i; /* 获取IB设备列表 */ dev_list = ibv_get_device_list(&num_devices); if (!dev_list) { perror("ibv_get_device_list"); return -1; } /* 选择第一个IB设备 */ ib_dev = dev_list[0]; if (!ib_dev) { fprintf(stderr, "No IB devices found\n"); return -1; } /* 打开IB设备 */ ctx.ctx = ibv_open_device(ib_dev); if (!ctx.ctx) { perror("ibv_open_device"); return -1; } /* 创建PD */ ctx.pd = ibv_alloc_pd(ctx.ctx); if (!ctx.pd) { perror("ibv_alloc_pd"); return -1; } /* 分配内存 */ ctx.rdma_buf = malloc(RDMA_BUF_SIZE); if (!ctx.rdma_buf) { perror("malloc"); return -1; } /* 注册内存 */ ctx.mr = ibv_reg_mr(ctx.pd, ctx.rdma_buf, RDMA_BUF_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!ctx.mr) { perror("ibv_reg_mr"); return -1; } /* 创建CQ */ ctx.cq = ibv_create_cq(ctx.ctx, 1, NULL, NULL, 0); if (!ctx.cq) { perror("ibv_create_cq"); return -1; } /* 创建QP */ memset(&qp_init_attr, 0, sizeof(qp_init_attr)); qp_init_attr.send_cq = ctx.cq; qp_init_attr.recv_cq = ctx.cq; qp_init_attr.qp_type = IBV_QPT_RC; qp_init_attr.cap.max_send_wr = 1; qp_init_attr.cap.max_recv_wr = 1; qp_init_attr.cap.max_send_sge = 1; qp_init_attr.cap.max_recv_sge = 1; ctx.qp = ibv_create_qp(ctx.pd, &qp_init_attr); if (!ctx.qp) { perror("ibv_create_qp"); return -1; } /* 修改QP状态 */ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; qp_attr.pkey_index = 0; qp_attr.port_num = 1; qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE; ret = ibv_modify_qp(ctx.qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS); if (ret) { perror("ibv_modify_qp"); return -1; } /* 获取端口属性 */ ret = ibv_query_port(ctx.ctx, 1, &ctx.port_attr); if (ret) { perror("ibv_query_port"); return -1; } /* 创建Completion Channel */ ctx.comp_channel = ibv_create_comp_channel(ctx.ctx); if (!ctx.comp_channel) { perror("ibv_create_comp_channel"); return -1; } /* 将CQ绑定到Completion Channel */ ret = ibv_req_notify_cq(ctx.cq, 0); if (ret) { perror("ibv_req_notify_cq"); return -1; } /* 等待CQ事件 */ ret = ibv_get_cq_event(ctx.comp_channel, &ctx.cq, &ctx.ctx); if (ret) { perror("ibv_get_cq_event"); return -1; } /* 请求下一个CQ事件 */ ret = ibv_req_notify_cq(ctx.cq, 0); if (ret) { perror("ibv_req_notify_cq"); return -1; } /* 获取远程节点的rkey和地址 */ ctx.rkey = 0x12345678; ctx.remote_addr = 0xdeadbeef; /* 向远程节点发送数据 */ memset(ctx.rdma_buf, 0, RDMA_BUF_SIZE); strcpy(ctx.rdma_buf, "Hello RDMA!"); struct ibv_send_wr wr, *bad_wr; struct ibv_sge sge; memset(&wr, 0, sizeof(wr)); wr.wr_id = 0; wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM; wr.send_flags = IBV_SEND_SIGNALED; wr.imm_data = 0x1234; wr.wr.rdma.remote_addr = ctx.remote_addr; wr.wr.rdma.rkey = ctx.rkey; wr.sg_list = &sge; wr.num_sge = 1; sge.addr = (uintptr_t)ctx.rdma_buf; sge.length = strlen(ctx.rdma_buf) + 1; sge.lkey = ctx.mr->lkey; ret = ibv_post_send(ctx.qp, &wr, &bad_wr); if (ret) { perror("ibv_post_send"); return -1; } /* 等待发送完成 */ do { ret = ibv_poll_cq(ctx.cq, 1, &wc); if (ret < 0) { perror("ibv_poll_cq"); return -1; } } while (ret == 0); /* 检查发送状态 */ if (wc.status != IBV_WC_SUCCESS) { fprintf(stderr, "Send failed with status %d\n", wc.status); return -1; } /* 关闭QP */ ret = ibv_destroy_qp(ctx.qp); if (ret) { perror("ibv_destroy_qp"); return -1; } /* 关闭Completion Channel */ ret = ibv_destroy_comp_channel(ctx.comp_channel); if (ret) { perror("ibv_destroy_comp_channel"); return -1; } /* 关闭CQ */ ret = ibv_destroy_cq(ctx.cq); if (ret) { perror("ibv_destroy_cq"); return -1; } /* 注销内存 */ ret = ibv_dereg_mr(ctx.mr); if (ret) { perror("ibv_dereg_mr"); return -1; } /* 释放内存 */ free(ctx.rdma_buf); /* 释放PD */ ret = ibv_dealloc_pd(ctx.pd); if (ret) { perror("ibv_dealloc_pd"); return -1; } /* 关闭IB设备 */ ret = ibv_close_device(ctx.ctx); if (ret) { perror("ibv_close_device"); return -1; } /* 释放IB设备列表 */ ibv_free_device_list(dev_list); return 0; } ```
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值