rdma-ud例程

最新推荐文章于 2024-05-19 02:11:47 发布
最新推荐文章于 2024-05-19 02:11:47 发布
阅读量526 收藏
点赞数
分类专栏： RDMA 文章标签： java 开发语言
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_38158479/article/details/127753482
版权 
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <endian.h>
#include <infiniband/verbs.h>

enum {
    PINGPONG_RECV_WRID = 1,
    PINGPONG_SEND_WRID = 2,
};

static int page_size;

struct pingpong_context {
    struct ibv_context  *context;
    struct ibv_comp_channel *channel;
    struct ibv_pd       *pd;
    struct ibv_mr       *mr;
    struct ibv_cq       *cq;
    struct ibv_qp       *qp;
    struct ibv_ah       *ah;
    char         *buf;
    int          size;
    int          send_flags;
    int          rx_depth;
    int          pending;
    struct ibv_port_attr     portinfo;
};

struct pingpong_dest {
    int lid;
    int qpn;
    int psn;
    union ibv_gid gid;
};

void
wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
{
    char tmp[9];
    __be32 v32;
    int i;
    uint32_t tmp_gid[4];

    for (tmp[8] = 0, i = 0; i < 4; ++i) {
        memcpy(tmp, wgid + i * 8, 8);
        sscanf(tmp, "%x", &v32);
        tmp_gid[i] = be32toh(v32);
    }
    memcpy(gid, tmp_gid, sizeof(*gid));
}

void
gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
{
    uint32_t tmp_gid[4];
    int i;

    memcpy(tmp_gid, gid, sizeof(tmp_gid));
    for (i = 0; i < 4; ++i)
        sprintf(&wgid[i * 8], "%08x", htobe32(tmp_gid[i]));
}

static struct pingpong_context *
init_ctx(struct ibv_device *ib_dev, int size, int port)
{
    struct pingpong_context *ctx = NULL;

    ctx = malloc(sizeof(*ctx));
    if(!ctx) {
        printf("malloc ctx error\n");
        return NULL;
    }

    ctx->size = size;
    ctx->send_flags = IBV_SEND_SIGNALED;
    ctx->buf = memalign(page_size, size + 40);
    if(!ctx->buf) {
        printf("ctx->buf malloc error\n");
        goto clean_ctx;
    }

    memset(ctx->buf, 0, size + 40);

    ctx->context = ibv_open_device(ib_dev);
    if(!ctx->context) {
        printf("ctx->context error, %s\n", ibv_get_device_name(ib_dev));
        goto clean_buffer;
    }

    {
        /*查询端口 mtu大小*/
        struct ibv_port_attr port_info = {};
        int mtu;
        if(ibv_query_port(ctx->context, port, &port_info)) {
            printf("ibv_query_port error\n");
            goto clean_device;
        }
        mtu = 1 << (port_info.active_mtu + 7);
        if (size > mtu) {
            printf("Requested size larger than port MTU (%d)\n", mtu);
            goto clean_device;
        }
    }

    ctx->channel = ibv_create_comp_channel(ctx->context);
    if(!ctx->channel) {
        printf("ibv_create_comp_channel error\n");
        goto clean_device;
    }

    ctx->pd = ibv_alloc_pd(ctx->context);
    if(!ctx->pd) {
        printf("ibv_alloc_pd error\n");
        goto clean_comp_channel;
    }

    ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
    if(!ctx->mr) {
        printf("ibv_reg_mr error\n");
        goto clean_pd;
    }

    ctx->cq = ibv_create_cq(ctx->context, 10, NULL, ctx->channel, 0);
    if(!ctx->cq) {
        printf("ibv_create_cq error\n");
        goto clean_mr;
    }

    {
        struct ibv_qp_attr attr;
        struct ibv_qp_init_attr init_attr = {
            .send_cq = ctx->cq,
            .recv_cq = ctx->cq,
            .cap     = {
                .max_send_wr  = 10,
                .max_recv_wr  = 10,
                .max_send_sge = 1,
                .max_recv_sge = 1
            },
            .qp_type = IBV_QPT_UD,
        };

        ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
        if (!ctx->qp)  {
            printf("Couldn't create QP\n");
            goto clean_cq;
        }

        ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
        if (init_attr.cap.max_inline_data >= size) {
            ctx->send_flags |= IBV_SEND_INLINE;
        }
    }

    {
        struct ibv_qp_attr attr = {
            .qp_state        = IBV_QPS_INIT,
            .pkey_index      = 0,
            .port_num        = port,
            .qkey            = 0x11111111
        };
        /*修改QP状态-->IBV_QPS_INIT*/
        if (ibv_modify_qp(ctx->qp, &attr,
                  IBV_QP_STATE              |
                  IBV_QP_PKEY_INDEX         |
                  IBV_QP_PORT               |
                  IBV_QP_QKEY)) {
            printf("Failed to modify QP to INIT\n");
            goto clean_qp;
        }
    }
    return ctx;
clean_qp:
    ibv_destroy_qp(ctx->qp);
clean_cq:
    ibv_destroy_cq(ctx->cq);
clean_mr:
    ibv_dereg_mr(ctx->mr);
clean_pd:
    ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
    if (ctx->channel)
        ibv_destroy_comp_channel(ctx->channel);
clean_device:
    ibv_close_device(ctx->context);
clean_buffer:
    free(ctx->buf);
clean_ctx:
    free(ctx);
    return NULL;
}

static int 
close_ctx(struct pingpong_context *ctx)
{
    if (ibv_destroy_qp(ctx->qp)) {
        fprintf(stderr, "Couldn't destroy QP\n");
        return 1;
    }

    if (ibv_destroy_cq(ctx->cq)) {
        fprintf(stderr, "Couldn't destroy CQ\n");
        return 1;
    }

    if (ibv_dereg_mr(ctx->mr)) {
        fprintf(stderr, "Couldn't deregister MR\n");
        return 1;
    }

    if (ibv_destroy_ah(ctx->ah)) {
        fprintf(stderr, "Couldn't destroy AH\n");
        return 1;
    }

    if (ibv_dealloc_pd(ctx->pd)) {
        fprintf(stderr, "Couldn't deallocate PD\n");
        return 1;
    }

    if (ctx->channel) {
        if (ibv_destroy_comp_channel(ctx->channel)) {
            fprintf(stderr, "Couldn't destroy completion channel\n");
            return 1;
        }
    }

    if (ibv_close_device(ctx->context)) {
        fprintf(stderr, "Couldn't release context\n");
        return 1;
    }
    free(ctx->buf);
    free(ctx);
    return 0;
}

static int
post_recv(struct pingpong_context *ctx)
{
    struct ibv_sge list = {
        .addr   = (uintptr_t) ctx->buf,
        .length = ctx->size + 40,
        .lkey   = ctx->mr->lkey
    };
    struct ibv_recv_wr wr = {
        .wr_id      = PINGPONG_RECV_WRID,
        .sg_list    = &list,
        .num_sge    = 1,
        .next       = NULL,
    };
    struct ibv_recv_wr *bad_wr;
    ibv_post_recv(ctx->qp, &wr, &bad_wr);
    return 0;
}

static int
post_send(struct pingpong_context *ctx, uint32_t qpn)
{
    struct ibv_sge list = {
        .addr   = (uintptr_t) ctx->buf + 40,
        .length = ctx->size,
        .lkey   = ctx->mr->lkey
    };
    struct ibv_send_wr wr = {
        .wr_id      = PINGPONG_SEND_WRID,
        .sg_list    = &list,
        .num_sge    = 1,
        .opcode     = IBV_WR_SEND,
        .send_flags = ctx->send_flags,
        .wr         = {
            .ud = {
                 .ah          = ctx->ah,
                 .remote_qpn  = qpn,
                 .remote_qkey = 0x11111111
             }
        }
    };
    struct ibv_send_wr *bad_wr;
    return ibv_post_send(ctx->qp, &wr, &bad_wr);
}

static int
connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
    int sl, struct pingpong_dest *dest, int sgid_idx)
{
    struct ibv_ah_attr ah_attr = {
        .is_global     = 0,
        .dlid          = dest->lid,
        .sl            = sl,
        .src_path_bits = 0,
        .port_num      = port
    };

    struct ibv_qp_attr attr = {
        .qp_state       = IBV_QPS_RTR
    };

    if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
        fprintf(stderr, "Failed to modify QP to RTR\n");
        return 1;
    }

    attr.qp_state = IBV_QPS_RTS;
    attr.sq_psn   = my_psn;

    if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN)) {
        fprintf(stderr, "Failed to modify QP to RTS\n");
        return 1;
    }

    if (dest->gid.global.interface_id) {
        ah_attr.is_global = 1;
        ah_attr.grh.hop_limit = 1;
        ah_attr.grh.dgid = dest->gid;
        ah_attr.grh.sgid_index = sgid_idx;
    }

    ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
    if (!ctx->ah) {
        fprintf(stderr, "Failed to create AH\n");
        return 1;
    }
    return 0;
}

static struct pingpong_dest *
client(const char *servername, int port, 
    const struct pingpong_dest *my_dest)
{
    int sockfd = -1;
    struct sockaddr_in serv_addr;
    char gid[33];
    struct pingpong_dest *rem_dest = NULL;
    char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
    if((sockfd = socket(AF_INET, SOCK_STREAM, 0))< 0) {
        printf("\n Error : Could not create socket \n");
        return NULL;
    }

    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(port);
    serv_addr.sin_addr.s_addr = inet_addr(servername);
    if(connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))<0) {
        printf("connect error\n");
        return NULL;
    }

    gid_to_wire_gid(&my_dest->gid, gid);
    sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
    if (write(sockfd, msg, sizeof msg) != sizeof msg) {
        fprintf(stderr, "Couldn't send local address\n");
        goto out;
    }

    if (read(sockfd, msg, sizeof msg) != sizeof msg ||
        write(sockfd, "done", sizeof "done") != sizeof "done") {
        perror("client read/write");
        fprintf(stderr, "Couldn't read/write remote address\n");
        goto out;
    }

    rem_dest = malloc(sizeof *rem_dest);
    if (!rem_dest)
        goto out;

    sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
    wire_gid_to_gid(gid, &rem_dest->gid);
out:
    close(sockfd);
    return rem_dest;
}

int main(int argc, char **argv)
{
    struct ibv_device      **dev_list;
    struct ibv_device      *ib_dev;
    struct pingpong_context *ctx;
    struct pingpong_dest my_dest, *rem_dest;
    unsigned int             port = 18515;
    int                      ib_port = 1;
    int                      gidx = 1;
    int                      sl = 0;
    unsigned int             size = 500;
    char                     gid[33];
    char                    *servername = NULL;
    srand48(getpid() * time(NULL));
    while (1) {
        int c;
        static struct option long_options[] = {
            { .name = "port",     .has_arg = 1, .val = 'p' },
            { .name = "ib-port",  .has_arg = 1, .val = 'i' },
            { .name = "size",     .has_arg = 1, .val = 's' },
            { .name = "sl",       .has_arg = 1, .val = 'l' },
            {}
        };

        c = getopt_long(argc, argv, "p:i:s:r:l", long_options, NULL);
        if (c == -1)
            break;

        switch (c) {
        case 'p':
            port = strtol(optarg, NULL, 0);
            if (port > 65535) {
                return 1;
            }
            break;
        case 'i':
            ib_port = strtol(optarg, NULL, 0);
            if (ib_port < 1) {
                return 1;
            }
            break;
        case 's':
            size = strtoul(optarg, NULL, 0);
            break;
        case 'l':
            sl = strtol(optarg, NULL, 0);
            break;
        default:
            return 1;
        }
    }

    /*获取IP*/
    if (optind == argc - 1)
        servername = strdup(argv[optind]);
    else if (optind < argc) {
        return -1;
    }

    page_size = sysconf(_SC_PAGESIZE);
    dev_list = ibv_get_device_list(NULL);
    if(!dev_list){
        printf("ibv_get_device_list error\n");
        return -1;
    } else {
        ib_dev = *dev_list;
        if (!ib_dev) {
            fprintf(stderr, "No IB devices found\n");
            return 1;
        }
    }

    ctx = init_ctx(ib_dev, size, ib_port);
    if(!ctx){
        printf("init_ctx error\n");
        return -1;
    }

    post_recv(ctx);
    if(ibv_req_notify_cq(ctx->cq, 0)){
        printf("ibv_req_notify_cq error\n");
        return -1;
    }
    /*获取lid信息*/
    if(ibv_query_port(ctx->context, ib_port, &ctx->portinfo)){
        printf("ibv_query_port error\n");
        return -1;
    }

    my_dest.lid = ctx->portinfo.lid;
    my_dest.qpn = ctx->qp->qp_num;
    my_dest.psn = lrand48() & 0xffffff;

    /*得到my_dest.gid的值*/
    if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
        printf("Could not get local gid for gid index %d\n", gidx);
        return -1;
    }

    /*将my_dest.gid的值转为点分十进制形式*/
    inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof(gid));
    printf("local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
           my_dest.lid, my_dest.qpn, my_dest.psn, gid);

    rem_dest = client(servername, port, &my_dest);
    if(!rem_dest){
        printf("client error\n");
        return -1;
    }

    inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof(gid));
    printf("remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
           rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);

    if(connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) {
        printf("connect_ctx \n");
        return -1;
    }

    strncpy(ctx->buf + 40, "viscore client", sizeof "visocre client");
    if (post_send(ctx, rem_dest->qpn)) {
        printf("Couldn't post send\n");
        return -1;
    }

    int event_num = 0;
    while (event_num < 2) {
        struct ibv_cq *cq;
        struct ibv_wc wc;
        void *ex_ctx;

        ibv_get_cq_event(ctx->channel, &cq, &ex_ctx);
        ibv_ack_cq_events(cq, 1);
        event_num++;
        ibv_req_notify_cq(cq, 0);

        while(ibv_poll_cq(cq, 1, &wc)) {
            if(wc.status != IBV_WC_SUCCESS){
                printf("wc.status != IBV_WC_SUCCESS\n");
            }
            if(wc.opcode & IBV_WC_RECV) {
                printf("IBV_WC_RECV ok, %s\n", ctx->buf + 40);
            }
            else if(wc.opcode == IBV_WC_SEND){
                printf("IBV_WC_SEND ok\n");
            }
        }
    }

    if(close_ctx(ctx)){
        printf("close_ctx error \n");
        return -1;
    }
    ibv_free_device_list(dev_list);
    free(rem_dest);
    return 0;
}

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <endian.h>
#include <infiniband/verbs.h>

enum {
    PINGPONG_RECV_WRID = 1,
    PINGPONG_SEND_WRID = 2,
};

static int page_size;

struct pingpong_context {
    struct ibv_context  *context;
    struct ibv_comp_channel *channel;
    struct ibv_pd       *pd;
    struct ibv_mr       *mr;
    struct ibv_cq       *cq;
    struct ibv_qp       *qp;
    struct ibv_ah       *ah;
    char         *buf;
    int          size;
    int          send_flags;
    int          rx_depth;
    int          pending;
    struct ibv_port_attr     portinfo;
};

struct pingpong_dest {
    int lid;
    int qpn;
    int psn;
    union ibv_gid gid;
};

void
wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
{
    char tmp[9];
    __be32 v32;
    int i;
    uint32_t tmp_gid[4];

    for (tmp[8] = 0, i = 0; i < 4; ++i) {
        memcpy(tmp, wgid + i * 8, 8);
        sscanf(tmp, "%x", &v32);
        tmp_gid[i] = be32toh(v32);
    }
    memcpy(gid, tmp_gid, sizeof(*gid));
}

void
gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
{
    uint32_t tmp_gid[4];
    int i;

    memcpy(tmp_gid, gid, sizeof(tmp_gid));
    for (i = 0; i < 4; ++i)
        sprintf(&wgid[i * 8], "%08x", htobe32(tmp_gid[i]));
}

static struct pingpong_context *
init_ctx(struct ibv_device *ib_dev, int size, int port)
{
    struct pingpong_context *ctx = NULL;

    ctx = malloc(sizeof(*ctx));
    if(!ctx) {
        printf("malloc ctx error\n");
        return NULL;
    }

    ctx->size = size;
    ctx->send_flags = IBV_SEND_SIGNALED;
    ctx->buf = memalign(page_size, size + 40);
    if(!ctx->buf) {
        printf("ctx->buf malloc error\n");
        goto clean_ctx;
    }

    memset(ctx->buf, 0, size + 40);
    strncpy(ctx->buf + 40, "viscore server", sizeof "visocre server");

    ctx->context = ibv_open_device(ib_dev);
    if(!ctx->context) {
        printf("ctx->context error, %s\n", ibv_get_device_name(ib_dev));
        goto clean_buffer;
    }

    {
        /*查询端口 mtu大小*/
        struct ibv_port_attr port_info = {};
        int mtu;
        if(ibv_query_port(ctx->context, port, &port_info)) {
            printf("ibv_query_port error\n");
            goto clean_device;
        }
        mtu = 1 << (port_info.active_mtu + 7);
        if (size > mtu) {
            printf("Requested size larger than port MTU (%d)\n", mtu);
            goto clean_device;
        }
    }

    ctx->channel = ibv_create_comp_channel(ctx->context);
    if(!ctx->channel) {
        printf("ibv_create_comp_channel error\n");
        goto clean_device;
    }

    ctx->pd = ibv_alloc_pd(ctx->context);
    if(!ctx->pd) {
        printf("ibv_alloc_pd error\n");
        goto clean_comp_channel;
    }

    ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
    if(!ctx->mr) {
        printf("ibv_reg_mr error\n");
        goto clean_pd;
    }

    ctx->cq = ibv_create_cq(ctx->context, 10, NULL, ctx->channel, 0);
    if(!ctx->cq) {
        printf("ibv_create_cq error\n");
        goto clean_mr;
    }

    {
        struct ibv_qp_attr attr;
        struct ibv_qp_init_attr init_attr = {
            .send_cq = ctx->cq,
            .recv_cq = ctx->cq,
            .cap     = {
                .max_send_wr  = 10,
                .max_recv_wr  = 10,
                .max_send_sge = 1,
                .max_recv_sge = 1
            },
            .qp_type = IBV_QPT_UD,
        };

        ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
        if (!ctx->qp)  {
            printf("Couldn't create QP\n");
            goto clean_cq;
        }

        ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
        if (init_attr.cap.max_inline_data >= size) {
            ctx->send_flags |= IBV_SEND_INLINE;
        }
    }

    {
        struct ibv_qp_attr attr = {
            .qp_state        = IBV_QPS_INIT,
            .pkey_index      = 0,
            .port_num        = port,
            .qkey            = 0x11111111
        };
        /*修改QP状态-->IBV_QPS_INIT*/
        if (ibv_modify_qp(ctx->qp, &attr,
                  IBV_QP_STATE              |
                  IBV_QP_PKEY_INDEX         |
                  IBV_QP_PORT               |
                  IBV_QP_QKEY)) {
            printf("Failed to modify QP to INIT\n");
            goto clean_qp;
        }
    }
    return ctx;
clean_qp:
    ibv_destroy_qp(ctx->qp);
clean_cq:
    ibv_destroy_cq(ctx->cq);
clean_mr:
    ibv_dereg_mr(ctx->mr);
clean_pd:
    ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
    if (ctx->channel)
        ibv_destroy_comp_channel(ctx->channel);
clean_device:
    ibv_close_device(ctx->context);
clean_buffer:
    free(ctx->buf);
clean_ctx:
    free(ctx);
    return NULL;
}

static int 
close_ctx(struct pingpong_context *ctx)
{
    if (ibv_destroy_qp(ctx->qp)) {
        fprintf(stderr, "Couldn't destroy QP\n");
        return 1;
    }

    if (ibv_destroy_cq(ctx->cq)) {
        fprintf(stderr, "Couldn't destroy CQ\n");
        return 1;
    }

    if (ibv_dereg_mr(ctx->mr)) {
        fprintf(stderr, "Couldn't deregister MR\n");
        return 1;
    }

    if (ibv_destroy_ah(ctx->ah)) {
        fprintf(stderr, "Couldn't destroy AH\n");
        return 1;
    }

    if (ibv_dealloc_pd(ctx->pd)) {
        fprintf(stderr, "Couldn't deallocate PD\n");
        return 1;
    }

    if (ctx->channel) {
        if (ibv_destroy_comp_channel(ctx->channel)) {
            fprintf(stderr, "Couldn't destroy completion channel\n");
            return 1;
        }
    }

    if (ibv_close_device(ctx->context)) {
        fprintf(stderr, "Couldn't release context\n");
        return 1;
    }
    free(ctx->buf);
    free(ctx);
    return 0;
}

static int
post_recv(struct pingpong_context *ctx)
{
    struct ibv_sge list = {
        .addr   = (uintptr_t) ctx->buf,
        .length = ctx->size + 40,
        .lkey   = ctx->mr->lkey
    };
    struct ibv_recv_wr wr = {
        .wr_id      = PINGPONG_RECV_WRID,
        .sg_list    = &list,
        .num_sge    = 1,
        .next       = NULL,
    };
    struct ibv_recv_wr *bad_wr;
    ibv_post_recv(ctx->qp, &wr, &bad_wr);
    return 0;
}

static int
post_send(struct pingpong_context *ctx, uint32_t qpn)
{
    struct ibv_sge list = {
        .addr   = (uintptr_t) ctx->buf + 40,
        .length = ctx->size,
        .lkey   = ctx->mr->lkey
    };
    struct ibv_send_wr wr = {
        .wr_id      = PINGPONG_SEND_WRID,
        .sg_list    = &list,
        .num_sge    = 1,
        .opcode     = IBV_WR_SEND,
        .send_flags = ctx->send_flags,
        .wr         = {
            .ud = {
                 .ah          = ctx->ah,
                 .remote_qpn  = qpn,
                 .remote_qkey = 0x11111111
             }
        }
    };
    struct ibv_send_wr *bad_wr;
    return ibv_post_send(ctx->qp, &wr, &bad_wr);
}

static int
connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
    int sl, struct pingpong_dest *dest, int sgid_idx)
{
    struct ibv_ah_attr ah_attr = {
        .is_global     = 0,
        .dlid          = dest->lid,
        .sl            = sl,
        .src_path_bits = 0,
        .port_num      = port
    };

    struct ibv_qp_attr attr = {
        .qp_state       = IBV_QPS_RTR
    };

    if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
        fprintf(stderr, "Failed to modify QP to RTR\n");
        return 1;
    }

    attr.qp_state = IBV_QPS_RTS;
    attr.sq_psn   = my_psn;

    if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN)) {
        fprintf(stderr, "Failed to modify QP to RTS\n");
        return 1;
    }

    if (dest->gid.global.interface_id) {
        ah_attr.is_global = 1;
        ah_attr.grh.hop_limit = 1;
        ah_attr.grh.dgid = dest->gid;
        ah_attr.grh.sgid_index = sgid_idx;
    }

    ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
    if (!ctx->ah) {
        fprintf(stderr, "Failed to create AH\n");
        return 1;
    }
    return 0;
}

static struct pingpong_dest *
server(struct pingpong_context *ctx, int ib_port, int port, 
    int sl, const struct pingpong_dest *my_dest, int sgid_idx)
{
    char gid[33];
    int n = 0;
    int sockfd = -1, connfd = -1;
    struct sockaddr_in serv_addr;
    struct pingpong_dest *rem_dest = NULL;
    char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if(sockfd < 0) {
        printf("socket error\n");
        return NULL;
    }

    memset(&serv_addr, 0, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    serv_addr.sin_port = htons(port); 

    bind(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
    if(listen(sockfd, 1) < 0) {
        printf("accept error\n");
        goto out;
    }

    connfd = accept(sockfd, (struct sockaddr*)NULL, NULL);
    if(connfd < 0) {
        printf("accept error\n");
        return NULL;
    }

    close(sockfd);
    n = read(connfd, msg, sizeof(msg));
    if(n != sizeof(msg)) {
        printf("n != sizeof(msg) error\n");
        goto out;
    }

    rem_dest = malloc(sizeof *rem_dest);
    if (!rem_dest) {
        printf("malloc rem_dest error \n");
        goto out;
    }
    /*从msg里面读取数据，按照一定格式写入rem_dest的各个变量里*/
    sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
    /*把gid的值写入rem_dest->gid*/
    wire_gid_to_gid(gid, &rem_dest->gid);
    /*创建AH*/
    if(connect_ctx(ctx, ib_port, my_dest->psn, 
        sl, rem_dest, sgid_idx)) {
        printf("connect_ctx error\n");
        free(rem_dest);
        rem_dest = NULL;
        goto out;
    }

    gid_to_wire_gid(&my_dest->gid, gid);
    /*把值按一定的格式写入msg*/
    sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);
    
    if(write(connfd, msg, sizeof(msg)) != sizeof(msg)) {
        printf("write error\n");
        free(rem_dest);
        rem_dest = NULL;
        goto out;
    }

    if(read(connfd, msg, sizeof(msg)) != sizeof "done") {
        printf("read error\n");
        free(rem_dest);
        rem_dest = NULL;
        goto out;
    }
out:
    close(connfd);
    return rem_dest;
}

int main(int argc, char **argv)
{
    struct ibv_device      **dev_list;
    struct ibv_device      *ib_dev;
    struct pingpong_context *ctx;
    struct pingpong_dest my_dest, *rem_dest;
    unsigned int             port = 18515;
    int                      ib_port = 1;
    int                      gidx = 1;
    int                      sl = 0;
    unsigned int             size = 500;
    char                     gid[33];

    srand48(getpid() * time(NULL));
    while (1) {
        int c;
        static struct option long_options[] = {
            { .name = "port",     .has_arg = 1, .val = 'p' },
            { .name = "ib-port",  .has_arg = 1, .val = 'i' },
            { .name = "size",     .has_arg = 1, .val = 's' },
            { .name = "sl",       .has_arg = 1, .val = 'l' },
            {}
        };

        c = getopt_long(argc, argv, "p:i:s:r:l", long_options, NULL);
        if (c == -1)
            break;

        switch (c) {
        case 'p':
            port = strtol(optarg, NULL, 0);
            if (port > 65535) {
                return 1;
            }
            break;
        case 'i':
            ib_port = strtol(optarg, NULL, 0);
            if (ib_port < 1) {
                return 1;
            }
            break;
        case 's':
            size = strtoul(optarg, NULL, 0);
            break;
        case 'l':
            sl = strtol(optarg, NULL, 0);
            break;
        default:
            return 1;
        }
    }

    page_size = sysconf(_SC_PAGESIZE);
    dev_list = ibv_get_device_list(NULL);
    if(!dev_list){
        printf("ibv_get_device_list error\n");
        return -1;
    } else {
        ib_dev = *dev_list;
        if (!ib_dev) {
            fprintf(stderr, "No IB devices found\n");
            return 1;
        }
    }

    ctx = init_ctx(ib_dev, size, ib_port);
    if(!ctx){
        printf("init_ctx error\n");
        return -1;
    }

    post_recv(ctx);
    if(ibv_req_notify_cq(ctx->cq, 0)){
        printf("ibv_req_notify_cq error\n");
        return -1;
    }
    /*获取lid信息*/
    if(ibv_query_port(ctx->context, ib_port, &ctx->portinfo)){
        printf("ibv_query_port error\n");
        return -1;
    }

    my_dest.lid = ctx->portinfo.lid;
    my_dest.qpn = ctx->qp->qp_num;
    my_dest.psn = lrand48() & 0xffffff;

    /*得到my_dest.gid的值*/
    if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
        printf("Could not get local gid for gid index %d\n", gidx);
        return -1;
    }

    /*将my_dest.gid的值转为点分十进制形式*/
    inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof(gid));
    printf("local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
           my_dest.lid, my_dest.qpn, my_dest.psn, gid);

    rem_dest = server(ctx, ib_port, port, sl, &my_dest, gidx);
    if(!rem_dest){
        printf("server error\n");
        return -1;
    }

    inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof(gid));
    printf("remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
           rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);

    int event_num = 0;
    while (event_num < 2) {
        struct ibv_cq *cq;
        struct ibv_wc wc;
        void *ex_ctx;

        ibv_get_cq_event(ctx->channel, &cq, &ex_ctx);
        ibv_ack_cq_events(cq, 1);
        event_num++;
        ibv_req_notify_cq(cq, 0);

        while(ibv_poll_cq(cq, 1, &wc)) {
            if(wc.status != IBV_WC_SUCCESS){
                printf("wc.status != IBV_WC_SUCCESS\n");
            }
            if(wc.opcode & IBV_WC_RECV) {
                printf("IBV_WC_RECV ok, %s\n", ctx->buf + 40);
                strncpy(ctx->buf + 40, "viscore server", sizeof "visocre server");
                if (post_send(ctx, rem_dest->qpn)) {
                    printf("Couldn't post send\n");
                    return 1;
                }
            }
            else if(wc.opcode == IBV_WC_SEND){
                printf("IBV_WC_SEND ok\n");
            }
        }
    }

    if(close_ctx(ctx)){
        printf("close_ctx error \n");
        return -1;
    }
    ibv_free_device_list(dev_list);
    free(rem_dest);
    return 0;
}

.PHONY: clean
CFLAGS  := -Wall -Werror -g
LD      := gcc
LDLIBS  := -lrdmacm -libverbs -lpthread
APPS    := client server
all: ${APPS}
clean:
	rm -f *.o ${APPS}

由于ud操作没有确认机制，所以wireshark抓包时没有看到ack包。
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述

酸菜。
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
【RDMA】infiniband提升Redis性能|UC和RC时延比较|RC和UD性能比较
小鱼菜鸟的博客
07-22 660
目录 infiniband应用到Redis之后的提升效果 WAN上InfiniBand传输服务的实验分析 RC和UD性能比较 UC和RC的区别 (https://www.mellanox.com/related-docs/prod_software/RDMA_Aware_Programming_user_manual.pdf) RDMA 架...
RDMA 架构与实践(技术详解（一）：RDMA概述)
bandaoyu的note
04-14 7609
RDMA 架构与实践 | https://houmin.cc/posts/454a90d3/ RDMA，即Remote Direct Memory Access，是一种绕过远程主机OS kernel访问其内存中数据的技术，概念源自于DMA技术。在 DMA 技术中，外部设备（PCIe 设备）能够绕过 CPU 直接访问host memory；而 RDMA 则是指外部设备能够绕过 CPU，不仅可以访问本地主机的内存，还能够访问另一台主机上的用户态内存。由于不经过操作系统，不仅节省了大量 CPU 资源......
【报错处理】ib_write_bw执行遇到Failed to modify QP 358 to RTR原因与解决办法？
技术札记
05-19 948
QP可以处于不同的状态，其中RTR（Ready to Receive）是QP状态之一，表示QP已经准备好接收消息。两个IB接口并未链接，但是两个IB接口的服务器是能够互通的时候会出现该问题。更改拓扑，让测试的两个port在IB上能够通。比如直连或者交换机处理。命令: ib_write_bw 1.1.1.1。
【RDMA】RDMA编程 和相关资料
bandaoyu的note
01-19 2507
creating Scatter Gather Elements https://www.openfabrics.org/downloads/Media/SC11/SC11_Writing_Application_Programs_for_RDMA_using_OFA_Software-v5.pdf https://www.cs.mtsu.edu/~waderholdt/6430/papers/ibverbs.pdf
RDMA RC UC UD
The Blog of Huststephen
12-11 4197
RC:面向连接的可靠服务 UC:面向连接的不可靠服务 UD:面向数据报的不可靠服务 面向连接 vs 面向数据报 相同点：两者的通信均包括双方QP对的参与 不同点：面向连接的通信若有N个节点与之通信，本机需要N个QP对；                面向数据报的通信可以做到N个节点与之通信，本机仅需一个QP队； one-sided RDMA VS two side
RDMA学习路线总结
05-07 5042
转自:https://my.oschina.net/SysuHuyh5LoveHqq/blog/842767 本博客仅作为分享本人学习rdma技术过程中一些经验和资料分享，若有错误之处，还请指教。如有侵犯版权问题,请立即通知本人,本人立即删除.特此声明! 简介--什么是rdma RDMA(Remote Direct Memory Access)技术全称远程直接数据存取。 ...
DMA(Direct Memory Access，直接内存存取)
Ubuntu 14.04配置nfs服务
06-17 463
DMA原理：DMA(Direct Memory Access，直接内存存取) 是所有现代电脑的重要特色，他允许不同速度的硬件装置来沟通，而不需要依于 CPU 的大量 中断 负载。否则，CPU 需要从 来源 把每一片段的资料复制到 暂存器，然后把他们再次写回到新的地方。在这个时间中，CPU 对于其他的工作来说就无法使用。 DMA 传输将数据从一个地址空间复制到另外一个地址空间。当 CPU 初始化这个...
k8s-rdma-sriov-dev-plugin:Kubernetes Rdma SRIOV 设备插件
05-31
k8s-rdma-sriov-dev-plugin ( ) 这是一个简单的 rdma 设备插件，支持 IB 和 RoCE SRIOV vHCA 和 HCA。 这也支持 Mellanox NIC 的 DPDK 应用程序。 这个插件作为 daemonset 运行。 它的容器镜像可以在 rdma/k8s-rdma-...
rdma-core-35.0.tar.gz
03-27
【rdma-core-35.0.tar.gz】这个压缩包文件包含了远程直接内存访问（RDMA）的核心库，版本为35.0。RDMA是一种高效的数据传输技术，它允许网络上的应用程序直接读写远程系统的内存，而无需通过操作系统内核，从而显著...
RDMA-CORE依赖库
最新发布
06-12
**RDMA-CORE依赖库详解** RDMA-CORE是一个核心库，它为InfiniBand Verbs（IBverbs）提供支持，这是一种用于实现Remote Direct Memory Access（远程直接内存访问）的技术。RDMA允许网络上的两个节点之间直接交换数据...
rdma-core-devel-22.4-5.el7.x86_64.rpm
12-10
官方离线安装包，亲测可用
rdma-core-22.4-6.el7_9.x86_64.rpm
12-10
官方离线安装包，亲测可用
基于RDMA高速网络的高性能分布式系统
weixin_45585364的博客
10-23 2156
基于RDMA高速网络的高性能分布式系统魏星达，陈榕，陈海波上海交通大学并行与分布式系统研究所，上海 200240摘要：高速的RDMA网络设备已经被广泛部署在现代数据中心。...
一周一论文（翻译 总结）—— [SOSP 18] LITE Kernel RDMA Support for Datacenter Applications : 一个LITE 内核支持的RDMA通信库
Master-TJ的个人博客
11-14 1938
目录 Abstract 1. Introduction 2. BACKGROUND AND ISSUES OF RDMA 2.1 Background on RDMA 2.2 RDMA in DataCenter Applications 2.3 Issue 1: Mismatch in Abstractions 2.4 Issue 2: Unscalable Performance...
读《FaSST: Fast, Scalable and Simple Distributed Transactions with Two-sided (RDMA) Datagram RPCs》
weixin_44162667的博客
07-05 1386
FaSST: Fast,Scalable and Simple Distributed Transactions with Two-sided Datagram RPCs1 Introduction2 Background2.1 Fast distributed transactions2.2 RDMA3 Choosing networking primitives3.1 Advantage of...
linux中pingpong测试程序的解读
lindorx的博客
02-03 2110
总代码放在最后。 抽象一点讲就是两个球员来回打球，一个发球，一个等待接球，最后计算完成指定数量的来回需要的时间。 详细一点： 程序定义了几个结构抽象出了pingpong游戏，如下： //乒乓游戏的球员 typedef struct { int table; int player; int count; pthread_mutex_t blocks[2]; pthread_t thread; } player_t; //打球的乒乓球台 typedef struct { in
11. RDMA之Shared Receive Queue
weixin_39094034的博客
10-31 427
全称为Shared Receive Queue，直译为共享接收队列。
基于RDMA和NVM的大数据系统一致性协议研究
weixin_45585364的博客
09-09 1000
基于RDMA和NVM的大数据系统一致性协议研究吴昊1,2, 陈康1,2, 武永卫1,2, 郑纬民1,21 清华大学计算机科学与技术系，北京 1000842 北京信息科学与...
rdma-core依赖下载
12-27
rdma-core是用于在远程直接内存访问（RDMA）技术上执行的用户态库。要下载rdma-core的依赖，首先需要检查系统上是否已经安装了必要的依赖，例如cmake、libudev、libnl等。可以使用系统包管理器（如apt、yum等）来...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值