rdma-read/write例程

于 2022-11-04 15:33:44 发布
阅读量422 收藏 3
点赞数 1
分类专栏： RDMA 文章标签： 网络 linux
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_38158479/article/details/127690112
版权

client.c

#include <stdio.h>
#include <netdb.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>

/*RDMA operation mode*/
enum mode {
    M_WRITE,
    M_READ
};

struct message {
    enum {
        MSG_MR,
        MSG_DONE
    } type;
    union {
        struct ibv_mr mr;
    } data;
};

struct context {
    struct ibv_context *ctx;
    struct ibv_pd *pd;
    struct ibv_cq *cq;
    struct ibv_comp_channel *comp_channel;
    pthread_t cq_poller_thread;
};

struct connection {
    struct rdma_cm_id *id;
    struct ibv_qp *qp;
    int connected;
    struct ibv_mr *recv_mr;
    struct ibv_mr *send_mr;
    struct ibv_mr *rdma_local_mr;
    struct ibv_mr *rdma_remote_mr;
    struct ibv_mr peer_mr;
    struct message *recv_msg;
    struct message *send_msg;
    char *rdma_local_region;
    char *rdma_remote_region;
    enum {
        SS_INIT,
        SS_MR_SENT,
        SS_RDMA_SENT,
        SS_DONE_SENT
    } send_state;
    enum {
        RS_INIT,
        RS_MR_RECV,
        RS_DONE_RECV
    } recv_state;
};

static struct context *s_ctx = NULL;
static enum mode s_mode = M_WRITE;
static const int RDMA_BUFFER_SIZE = 1024;
const int TIMEOUT_IN_MS = 500;

char *
get_peer_message_region(struct connection *conn)
{
    if (s_mode == M_WRITE)
        return conn->rdma_remote_region;
    else
        return conn->rdma_local_region;
}

void
completion(struct ibv_wc *wc)
{
    struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
    if(wc->status != IBV_WC_SUCCESS) {
        printf("wc->status != IBV_WC_SUCCESS");
    }

    if(wc->opcode & IBV_WC_RECV) {
        conn->recv_state++;

        if(conn->recv_msg->type == MSG_MR) {
            /*收到服务端发送过来的mr,key等信息*/
            memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
            struct ibv_recv_wr wr, *bad_wr = NULL;
            struct ibv_sge sge;

            wr.wr_id = (uintptr_t)conn;
            wr.next = NULL;
            wr.sg_list = &sge;
            wr.num_sge = 1;
            sge.addr = (uintptr_t)conn->recv_msg;
            sge.length = sizeof(struct message);
            sge.lkey = conn->recv_mr->lkey;
            /*预留recv buffer*/
            ibv_post_recv(conn->qp, &wr, &bad_wr);
        }
    } else {
        conn->send_state++;
    }

    /*客户端经历了一次recv和一次send以后就会进来*/
    if((conn->send_state == SS_MR_SENT) && (conn->recv_state == RS_MR_RECV)) {
        struct ibv_send_wr wr, *bad_wr = NULL;
        struct ibv_sge sge;

        memset(&wr, 0, sizeof(wr));
        wr.wr_id = (uintptr_t)conn;
        wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
        wr.sg_list = &sge;
        wr.num_sge = 1;
        wr.send_flags = IBV_SEND_SIGNALED;
        wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
        wr.wr.rdma.rkey = conn->peer_mr.rkey;
        sge.addr = (uintptr_t)conn->rdma_local_region;
        sge.length = RDMA_BUFFER_SIZE;
        sge.lkey = conn->rdma_local_mr->lkey;
        /*假如是写模式，客户端把conn->rdma_local_region缓冲区的内容写入到服务端，
        也就是写入conn->peer_mr.addr，如果是读模式，则把服务端的conn->peer_mr.addr
        地址的内容写入到rdma_local_region*/
        ibv_post_send(conn->qp, &wr, &bad_wr);

        /*结束了，为了能走进rdma_disconnect*/
        conn->send_msg->type = MSG_DONE;
        memset(&wr, 0, sizeof(wr));
        wr.wr_id = (uintptr_t)conn;
        wr.opcode = IBV_WR_SEND;
        wr.sg_list = &sge;
        wr.num_sge = 1;
        wr.send_flags = IBV_SEND_SIGNALED;
        sge.addr = (uintptr_t)conn->send_msg;
        sge.length = sizeof(struct message);
        sge.lkey = conn->send_mr->lkey;
        while (!conn->connected);
        ibv_post_send(conn->qp, &wr, &bad_wr);
    } else if((conn->send_state == SS_DONE_SENT) && (conn->recv_state == RS_DONE_RECV)) {
        printf("remote buffer: %s\n", get_peer_message_region(conn));
        rdma_disconnect(conn->id);
    }
    return;
}

void *
poll_cq(void *ctx)
{
    struct ibv_cq *cq;
    struct ibv_wc wc;

    while (1) {
    ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx);
    ibv_ack_cq_events(cq, 1);
    ibv_req_notify_cq(cq, 0);
    while (ibv_poll_cq(cq, 1, &wc))
        completion(&wc);
    }
    return NULL;
}

void*
get_local_message_region(void *context)
{
    if (s_mode == M_WRITE)
        return ((struct connection *)context)->rdma_local_region;
    else
        return ((struct connection *)context)->rdma_remote_region;
}

int
resolved(struct rdma_cm_id *id)
{
    struct connection *conn;
    struct ibv_qp_init_attr qp_attr;
    struct ibv_recv_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    if(s_ctx) {
        if (s_ctx->ctx != id->verbs) {
            printf("cannot handle events in more than one context\n");
            return -1;
        }
    }

    s_ctx = (struct context *)malloc(sizeof(struct context));
    s_ctx->ctx = id->verbs;

    s_ctx->pd = ibv_alloc_pd(s_ctx->ctx);
    s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx);
    s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0);
    ibv_req_notify_cq(s_ctx->cq, 0);

    pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL);

    memset(&qp_attr, 0, sizeof(qp_attr));
    qp_attr.send_cq = s_ctx->cq;
    qp_attr.recv_cq = s_ctx->cq;
    qp_attr.qp_type = IBV_QPT_RC;
    qp_attr.cap.max_send_wr = 10;
    qp_attr.cap.max_recv_wr = 10;
    qp_attr.cap.max_send_sge = 1;
    qp_attr.cap.max_recv_sge = 1;

    if(rdma_create_qp(id, s_ctx->pd, &qp_attr)) {
        printf("rdma_create_qp error\n");
    }

    id->context = conn = (struct connection *)malloc(sizeof(struct connection));
    conn->id = id;
    conn->qp = id->qp;

    conn->send_state = SS_INIT;
    conn->recv_state = RS_INIT;
    conn->connected = 0;

    conn->send_msg = malloc(sizeof(struct message));
    conn->recv_msg = malloc(sizeof(struct message));

    conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
    conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);

    conn->send_mr = ibv_reg_mr(s_ctx->pd, conn->send_msg, sizeof(struct message), 0);
    if(!conn->send_mr){
        printf("ibv_reg_mr send_mr error\n");
    }

    conn->recv_mr = ibv_reg_mr( s_ctx->pd, conn->recv_msg, sizeof(struct message),
            IBV_ACCESS_LOCAL_WRITE);
    if(!conn->recv_mr){
        printf("ibv_reg_mr recv_mr error\n");
    }

    conn->rdma_local_mr = ibv_reg_mr(s_ctx->pd, conn->rdma_local_region, RDMA_BUFFER_SIZE, 
        ((s_mode == M_WRITE) ? 0 : IBV_ACCESS_LOCAL_WRITE));
    if(!conn->rdma_local_mr) {
            printf("ibv_reg_mr local_mr error\n");
    }

    conn->rdma_remote_mr = ibv_reg_mr(s_ctx->pd, conn->rdma_remote_region, 
        RDMA_BUFFER_SIZE, ((s_mode == M_WRITE) ? (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE) : IBV_ACCESS_REMOTE_READ));
    if(!conn->rdma_remote_mr) {
        printf("ibv_reg_mr rdma_remote_mr error\n");
    }

    wr.wr_id = (uintptr_t)conn;
    wr.next = NULL;
    wr.sg_list = &sge;
    wr.num_sge = 1;

    sge.addr = (uintptr_t)conn->recv_msg;
    sge.length = sizeof(struct message);
    sge.lkey = conn->recv_mr->lkey;

    /*客户端先准备好recv buffer*/
    if(ibv_post_recv(conn->qp, &wr, &bad_wr)){
        printf("ibv_post_recv error\n");
    }

    sprintf(get_local_message_region(id->context), "message from active/client side with pid %d", getpid());

    if(rdma_resolve_route(id, TIMEOUT_IN_MS)) {
        printf("rdma_resolve_route error\n");
    }
    return 0;
}

int
route(struct rdma_cm_id *id)
{
    struct rdma_conn_param params;
    memset(&params, 0, sizeof(params));

    params.initiator_depth = params.responder_resources = 1;
    params.rnr_retry_count = 7;
    /*客户端发起连接*/
    rdma_connect(id, &params);
    return 0;
}

int
connection(struct rdma_cm_id *id)
{
    struct ibv_send_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    ((struct connection *)id->context)->connected = 1;
    struct connection *conn = (struct connection *)id->context;

    /*把rdma_remote_mr等信息拷贝到send_msg*/
    conn->send_msg->type = MSG_MR;
    memcpy(&conn->send_msg->data.mr, conn->rdma_remote_mr, sizeof(struct ibv_mr));

    memset(&wr, 0, sizeof(wr));
    wr.wr_id = (uintptr_t)conn;
    wr.opcode = IBV_WR_SEND;
    wr.sg_list = &sge;
    wr.num_sge = 1;
    wr.send_flags = IBV_SEND_SIGNALED;

    sge.addr = (uintptr_t)conn->send_msg;
    sge.length = sizeof(struct message);
    sge.lkey = conn->send_mr->lkey;

    while(!conn->connected);

    /*服务端接受请求以后进入该函数，客户端把实现准备的好mr,key等信息下发给服务端*/
    if(ibv_post_send(conn->qp, &wr, &bad_wr)) {
        printf("ibv_post_send error\n");
    }
    return 0;
}

int
disconnect(struct rdma_cm_id *id)
{
    struct connection *conn = (struct connection *)id->context;

    rdma_destroy_qp(conn->id);
    ibv_dereg_mr(conn->send_mr);
    ibv_dereg_mr(conn->recv_mr);
    ibv_dereg_mr(conn->rdma_local_mr);
    ibv_dereg_mr(conn->rdma_remote_mr);
    free(conn->send_msg);
    free(conn->recv_msg);
    free(conn->rdma_local_region);
    free(conn->rdma_remote_region);
    rdma_destroy_id(conn->id);
    free(conn);
    return 1;
}

int
client_event(struct rdma_cm_event *event)
{
    int ret = 0;
    if (event->event == RDMA_CM_EVENT_ADDR_RESOLVED) {
        ret = resolved(event->id);
        printf("RDMA_CM_EVENT_ADDR_RESOLVED\n");
    } else if (event->event == RDMA_CM_EVENT_ROUTE_RESOLVED) {
        ret = route(event->id);
        printf("RDMA_CM_EVENT_ROUTE_RESOLVED\n");
    } else if (event->event == RDMA_CM_EVENT_ESTABLISHED) {
        ret = connection(event->id);
        printf("RDMA_CM_EVENT_ESTABLISHED\n");
    } else if (event->event == RDMA_CM_EVENT_DISCONNECTED) {
        ret = disconnect(event->id);
        printf("RDMA_CM_EVENT_DISCONNECTED\n");
    } else {
        fprintf(stdout, "on_event: %d\n", event->event);
    }
    return ret;
}

int main(int argc, char **argv)
{
    struct addrinfo *addr;
    struct rdma_cm_event *event = NULL;
    struct rdma_cm_id *conn = NULL;
    struct rdma_event_channel *ec = NULL;

    if(argc != 4) {
        fprintf(stdout, "usage:<mode> <server-address> <server-port>\n");
        exit(-1);
    }

    if(!strcmp(argv[1], "write")) {
        s_mode = M_WRITE;
    } else if(!strcmp(argv[1], "read")) {
        s_mode = M_READ;
    } else {
        fprintf(stdout, "usage:<mode> <server-address> <server-port>\n");
        exit(-1);
    }

    if(getaddrinfo(argv[2], argv[3], NULL, &addr)) {
        printf("getaddrinfo error\n");
    }

    ec = rdma_create_event_channel();
    if(!ec) {
        printf("rdma_create_event_channel error\n");
    }

    if(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)) {
        printf("rdma_create_id error\n");
    }

    if(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)) {
        printf("rdma_resolve_addr error\n");
    }
    freeaddrinfo(addr);

    while(!rdma_get_cm_event(ec, &event)) {
        struct rdma_cm_event event_copy;
        memcpy(&event_copy, event, sizeof(*event));
        rdma_ack_cm_event(event);

        if(client_event(&event_copy))
            break;
    }
    rdma_destroy_event_channel(ec);
    return 0;
}

server.c

#include <stdio.h>
#include <netdb.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>

enum mode {
    M_WRITE,
    M_READ
};

struct message {
    enum {
        MSG_MR,
        MSG_DONE
    }type;
    union {
        struct ibv_mr mr;
    }data;
};

struct context {
    struct ibv_context *ctx;
    struct ibv_pd *pd;
    struct ibv_cq *cq;
    struct ibv_comp_channel *comp_channel;
    pthread_t cq_poller_thread;
};

struct connection {
    struct rdma_cm_id *id;
    struct ibv_qp *qp;
    int connected;
    struct ibv_mr *recv_mr;
    struct ibv_mr *send_mr;
    struct ibv_mr *rdma_local_mr;
    struct ibv_mr *rdma_remote_mr;
    struct ibv_mr peer_mr;
    struct message *recv_msg;
    struct message *send_msg;
    char *rdma_local_region;
    char *rdma_remote_region;
    enum {
        SS_INIT,
        SS_MR_SENT,
        SS_RDMA_SENT,
        SS_DONE_SENT
    } send_state;
    enum {
        RS_INIT,
        RS_MR_RECV,
        RS_DONE_RECV
    } recv_state;
};

static const int RDMA_BUFFER_SIZE = 1024;
static struct context *s_ctx = NULL;
static enum mode s_mode = M_WRITE;

char *
get_peer_message_region(struct connection *conn)
{
    if (s_mode == M_WRITE)
        return conn->rdma_remote_region;
    else
        return conn->rdma_local_region;
}

void
completion(struct ibv_wc *wc)
{
    struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id;
    if(wc->status != IBV_WC_SUCCESS) {
        printf("wc->status != IBV_WC_SUCCESS\n");
    }
    /*客户端建立链接以后发起ibv_post_send服务端进入该判断语句*/
    if(wc->opcode & IBV_WC_RECV) {
        conn->recv_state++;

        if(conn->recv_msg->type == MSG_MR) {
            /*得到客户端发送的mr, key等信息*/
            memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr));
            struct ibv_recv_wr wr, *bad_wr = NULL;
            struct ibv_sge sge;

            wr.wr_id = (uintptr_t)conn;
            wr.next = NULL;
            wr.sg_list = &sge;
            wr.num_sge = 1;
            sge.addr = (uintptr_t)conn->recv_msg;
            sge.length = sizeof(struct message);
            sge.lkey = conn->recv_mr->lkey;
            /*进来用掉了一个，再下发一个ibv_post_recv，预防后面没有recv buffer*/
            ibv_post_recv(conn->qp, &wr, &bad_wr);

            /*服务端下发ibv_post_send*/
            if(conn->send_state == SS_INIT) {
                printf("server \n");
                struct ibv_send_wr wr, *bad_wr = NULL;
                struct ibv_sge sge;
                conn->send_msg->type = MSG_MR;
                memcpy(&conn->send_msg->data.mr, conn->rdma_remote_mr, sizeof(struct ibv_mr));

                memset(&wr, 0, sizeof(wr));
                wr.wr_id = (uintptr_t)conn;
                wr.opcode = IBV_WR_SEND;
                wr.sg_list = &sge;
                wr.num_sge = 1;
                wr.send_flags = IBV_SEND_SIGNALED;
                sge.addr = (uintptr_t)conn->send_msg;
                sge.length = sizeof(struct message);
                sge.lkey = conn->send_mr->lkey;
                while (!conn->connected);
                /*发送到客户端的recv buffer, 里面包含了服务端mr key等*/
                ibv_post_send(conn->qp, &wr, &bad_wr);
            }
        }
    } else {
        conn->send_state++;
    }

    /*服务端经历了一次recv和一次send以后就会进来*/
    if((conn->send_state == SS_MR_SENT) && (conn->recv_state == RS_MR_RECV)) {
        struct ibv_send_wr wr, *bad_wr = NULL;
        struct ibv_sge sge;

        memset(&wr, 0, sizeof(wr));
        wr.wr_id = (uintptr_t)conn;
        wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ;
        wr.sg_list = &sge;
        wr.num_sge = 1;
        wr.send_flags = IBV_SEND_SIGNALED;
        wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr;
        wr.wr.rdma.rkey = conn->peer_mr.rkey;
        sge.addr = (uintptr_t)conn->rdma_local_region;
        sge.length = RDMA_BUFFER_SIZE;
        sge.lkey = conn->rdma_local_mr->lkey;
        /*假如是写模式，服务端把conn->rdma_local_region缓冲区的内容写入到客户端，
        也就是写入conn->peer_mr.addr，如果是读模式，则把客户端的conn->peer_mr.addr
        地址的内容写入到rdma_local_region*/
        ibv_post_send(conn->qp, &wr, &bad_wr);

        /*结束了，为了能走进rdma_disconnect*/
        conn->send_msg->type = MSG_DONE;
        memset(&wr, 0, sizeof(wr));
        wr.wr_id = (uintptr_t)conn;
        wr.opcode = IBV_WR_SEND;
        wr.sg_list = &sge;
        wr.num_sge = 1;
        wr.send_flags = IBV_SEND_SIGNALED;
        sge.addr = (uintptr_t)conn->send_msg;
        sge.length = sizeof(struct message);
        sge.lkey = conn->send_mr->lkey;
        while (!conn->connected);
        ibv_post_send(conn->qp, &wr, &bad_wr);
    } else if((conn->send_state == SS_DONE_SENT) && (conn->recv_state == RS_DONE_RECV)) {
        printf("remote buffer: %s\n", get_peer_message_region(conn));
        rdma_disconnect(conn->id);
    }
    return;
}

void *
poll_cq(void *ctx)
{
    struct ibv_cq *cq;
    struct ibv_wc wc;

    while (1) {
    ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx);
    ibv_ack_cq_events(cq, 1);
    ibv_req_notify_cq(cq, 0);
    while (ibv_poll_cq(cq, 1, &wc))
        completion(&wc);
    }
    return NULL;
}

void *
get_local_message_region(void *context)
{
    if (s_mode == M_WRITE)
        return ((struct connection *)context)->rdma_local_region;
    else
        return ((struct connection *)context)->rdma_remote_region;
}

int
connect_request(struct rdma_cm_id *id)
{
    struct rdma_conn_param cm_params;
    struct connection *conn;
    struct ibv_qp_init_attr qp_attr;
    struct ibv_recv_wr wr, *bad_wr = NULL;
    struct ibv_sge sge;

    if(s_ctx) {
        if (s_ctx->ctx != id->verbs) {
            printf("cannot handle events in more than one context\n");
            return -1;
        }
    }

    s_ctx = (struct context *)malloc(sizeof(struct context));
    s_ctx->ctx = id->verbs;
    s_ctx->pd = ibv_alloc_pd(s_ctx->ctx);
    s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx);
    s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0);
    ibv_req_notify_cq(s_ctx->cq, 0);

    pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL);

    memset(&qp_attr, 0, sizeof(qp_attr));
    qp_attr.send_cq = s_ctx->cq;
    qp_attr.recv_cq = s_ctx->cq;
    qp_attr.qp_type = IBV_QPT_RC;
    qp_attr.cap.max_send_wr = 10;
    qp_attr.cap.max_recv_wr = 10;
    qp_attr.cap.max_send_sge = 1;
    qp_attr.cap.max_recv_sge = 1;

    if(rdma_create_qp(id, s_ctx->pd, &qp_attr)) {
        printf("rdma_create_qp error\n");
    }

    id->context = conn = (struct connection *)malloc(sizeof(struct connection));
    conn->id = id;
    conn->qp = id->qp;

    conn->send_state = SS_INIT;
    conn->recv_state = RS_INIT;
    conn->connected = 0;

    conn->send_msg = malloc(sizeof(struct message));
    conn->recv_msg = malloc(sizeof(struct message));
    conn->rdma_local_region = malloc(RDMA_BUFFER_SIZE);
    conn->rdma_remote_region = malloc(RDMA_BUFFER_SIZE);

    conn->send_mr = ibv_reg_mr(s_ctx->pd, conn->send_msg, sizeof(struct message), 0);
    if(!conn->send_mr){
        printf("ibv_reg_mr send_mr error\n");
    }

    conn->recv_mr = ibv_reg_mr( s_ctx->pd, conn->recv_msg, sizeof(struct message),
            IBV_ACCESS_LOCAL_WRITE);
    if(!conn->recv_mr){
        printf("ibv_reg_mr recv_mr error\n");
    }

    conn->rdma_local_mr = ibv_reg_mr(s_ctx->pd, conn->rdma_local_region, RDMA_BUFFER_SIZE, 
        ((s_mode == M_WRITE) ? 0 : IBV_ACCESS_LOCAL_WRITE));
    if(!conn->rdma_local_mr) {
            printf("ibv_reg_mr local_mr error\n");
    }

    conn->rdma_remote_mr = ibv_reg_mr(s_ctx->pd, conn->rdma_remote_region, 
        RDMA_BUFFER_SIZE, ((s_mode == M_WRITE) ? (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE) : IBV_ACCESS_REMOTE_READ));
    if(!conn->rdma_remote_mr) {
        printf("ibv_reg_mr rdma_remote_mr error\n");
    }

    wr.wr_id = (uintptr_t)conn;
    wr.next = NULL;
    wr.sg_list = &sge;
    wr.num_sge = 1;

    sge.addr = (uintptr_t)conn->recv_msg;
    sge.length = sizeof(struct message);
    sge.lkey = conn->recv_mr->lkey;

    if(ibv_post_recv(conn->qp, &wr, &bad_wr)){
        printf("ibv_post_recv error\n");
    }

    memset(&cm_params, 0, sizeof(cm_params));
    cm_params.initiator_depth = cm_params.responder_resources = 1;
    cm_params.rnr_retry_count = 7;

    sprintf(get_local_message_region(id->context), "message from passive/server side with pid %d", getpid());

    if(rdma_accept(id, &cm_params)) {
        printf("rdma_accept error\n");
    }
    return 0;
}

int
connection(struct rdma_cm_id *id)
{
    ((struct connection *)id->context)->connected = 1;
    return 0;
}

int
disconnect(struct rdma_cm_id *id)
{
    struct connection *conn = (struct connection *)id->context;

    rdma_destroy_qp(conn->id);
    ibv_dereg_mr(conn->send_mr);
    ibv_dereg_mr(conn->recv_mr);
    ibv_dereg_mr(conn->rdma_local_mr);
    ibv_dereg_mr(conn->rdma_remote_mr);
    free(conn->send_msg);
    free(conn->recv_msg);
    free(conn->rdma_local_region);
    free(conn->rdma_remote_region);
    rdma_destroy_id(conn->id);
    free(conn);
    return 0;
}

int
server_event(struct rdma_cm_event *event)
{
    int ret = 0;

    if(event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
        printf("RDMA_CM_EVENT_CONNECT_REQUEST\n");
        ret = connect_request(event->id);
    }
    else if (event->event == RDMA_CM_EVENT_ESTABLISHED) {
        printf("RDMA_CM_EVENT_ESTABLISHED\n");
        ret = connection(event->id);
    }
    else if (event->event == RDMA_CM_EVENT_DISCONNECTED) {
        printf("RDMA_CM_EVENT_DISCONNECTED\n");
        ret = disconnect(event->id);
    }
    else
        printf("unknown event\n");
    return ret;
}

int main(int argc, char **argv)
{
    struct sockaddr_in6 addr;
    struct rdma_cm_event *event = NULL;
    struct rdma_cm_id *listener = NULL;
    struct rdma_event_channel *ec = NULL;
    uint16_t port = 0;

    if(argc != 2) {
        printf("usage: mode\n");
        exit(-1);
    }

    if(!strcmp(argv[1], "write")) {
        s_mode = M_WRITE;
    } else if(!strcmp(argv[1], "read")) {
        s_mode = M_READ;
    } else {
        printf("usage: mode\n");
    }

    memset(&addr, 0, sizeof(addr));
    addr.sin6_family = AF_INET6;

    ec = rdma_create_event_channel();
    if(!ec) {
        printf("rdma_create_event_channel error\n");
    }

    if(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP)) {
        printf("rdma_create_id error\n");
    }

    if(rdma_bind_addr(listener, (struct sockaddr *)&addr)){
        printf("rdma_bind_addr error\n");
    }

    if(rdma_listen(listener, 10)) {
        printf("rdma_listen error\n");
    }

    port = ntohs(rdma_get_src_port(listener));
    printf("listening on port %d.\n", port);

    while(!rdma_get_cm_event(ec, &event)) {
        struct rdma_cm_event event_copy;
        memcpy(&event_copy, event, sizeof(*event));
        rdma_ack_cm_event(event);
        if(server_event(&event_copy))
            break;
    }

    rdma_destroy_id(listener);
    rdma_destroy_event_channel(ec);
    return 0;
}

Makefile

.PHONY: clean
CFLAGS  := -Wall -Werror -g
LD      := gcc
LDLIBS  := -lrdmacm -libverbs -lpthread
APPS    := client server
all: ${APPS}
clean:
	rm -f *.o ${APPS}

在这里插入图片描述
在这里插入图片描述
在这里插入图片描述

酸菜。
关注
  • 1
    点赞
  • 踩
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
RDMA read/write with ib verbs
684的博客
02-27 764
RDMA因为其kernel-bypass，zero-copy，cpu offload三种特性，能够提供低延迟、高吞吐的数据传输性能。 使用RDMA进行数据传输时，read/write在client/server端的具体操作可以概括为以下步骤： 1. 初始化连接和注册内存区(Memory Region); 2. 建立连接; 3. ...
RDMA send/recv 和 read/write操作
大隐隐于野
08-15 700
相当于说，我是client，对应id，我要发送的数据在addr放着，长度length，这个发送的数据所在的内存已经在RDMA设备上注册过了，是mr，发送标志是flags，控制写操作，由于我是直接写到服务器，服务器什么都不做，所以我得知道存到哪，而这个地址是通过send/recv交换过的，所以我就把数据放到remote_addr地址处，当然这块地址也是已经注册过的，我有这个地址的钥匙相当于操作权限rkey，这个也是服务器给我的。本地缓冲区和远程内存都是已经注册过的，才能和RDMA设备直接交互。
RDMA编程实例源码（rdma_cm API）
05-12
RDMA编程实例源码（rdma_cm API）
RDMA技术浅析（二）
袁赟的博客
04-08 1936
本章主要探讨RDMA软件相关的部分。 一、名词解释 首先解释一下几个名词： rdma-core 指开源RDMA用户态软件协议栈，包含用户态框架、各厂商用户态驱动、API帮助手册以及开发自测试工具等。 rdma-core在github上维护，我们的用户态Verbs API实际上就是它实现的。https://github.com/linux-rdma/rdma-core 代码目录结构如下： 其中比较重要的几个目录是： libibverbs 以ibv为前缀，这里的ib并不代表infiniband
RDMA
Likes的博客
10-19 2656
来源： https://zcopy.wordpress.com/ 说明： 本文不是对原文的逐字逐句翻译，而是摘取核心部分以介绍RDMA Send操作(后面凡是提到RDMA send, 都对应于IBA里的send操作)。文中给出的例子非常浅显易懂，很值得一读。 1. What is RDMA | 什么是RDMA RDMA is Remote Direct Memory Access whic...
【RDMA】RDMA SEND/WRITE编程实例（IBV Verbs ）
bandaoyu的note
04-22 2606
目录 基于Verbs的RDMA RC通信编程示例概要 Main print_config resources_init resources_create sock_connect connect_qp modify_qp_to_init post_receive sock_sync_data modify_qp_to_rtr modify_qp_to_rts post_send poll_completion resources_destroy 代码（Send, Rec.
rdma-core/irdma: Implement device supported verb APIs
07-12
rdma-core/irdma: Implement device supported verb APIs
rdma-core/irdma: Add library setup and utility
07-12
rdma-core/irdma: Add library setup and utility
rdma-core/irdma: Add user/kernel shared libraries
07-12
rdma-core/irdma: Add user/kernel shared libraries
k8s-rdma-sriov-dev-plugin:Kubernetes Rdma SRIOV 设备插件
05-31
k8s-rdma-sriov-dev-plugin ( ) 这是一个简单的 rdma 设备插件，支持 IB 和 RoCE SRIOV vHCA 和 HCA。 这也支持 Mellanox NIC 的 DPDK 应用程序。 这个插件作为 daemonset 运行。 它的容器镜像可以在 rdma/k8s-rdma-...
RDMA编程手册11111
09-21
RDMA_Aware_Programming_user_manual1.7.pdf,最新手册，之前在csdn下载了一个，太旧了，上传一个最新的供大家实用。手册介绍ofed相关接口用法，最详细，没有之一。
rdma-core-devel-22.4-5.el7.x86_64.rpm
12-10
官方离线安装包，亲测可用
RDMA 编程备忘录
abc593043305的博客
03-29 526
1、RDMA WRITE WITH IMM操作需要消耗receive request，因此在接收方处理完之后需要ibv_post_recv()避免RNR错误。 （测试所得，不准确之处请指教）
【RDMA】技术详解（一）：RDMA概述
热门推荐
bandaoyu的note
01-20 5万+
原文：https://blog.csdn.net/qq_21125183/article/details/86522475 1. DMA和RDMA概念 1.1 DMA DMA(直接内存访问)是一种能力，允许在计算机主板上的设备直接把数据发送到内存中去，数据搬运不需要CPU的参与。 传统内存访问需要通过CPU进行数据copy来移动数据，通过CPU将内存中的Buffer1移动到Buffer2中。DMA模式：可以同DMA Engine之间通过硬件将数据从Buffer1移动到Buffer2,而不需要操作系统
[SPDK/NVMe存储技术分析]009 - Introduction to RDMA Send | RDMA Send操作概论
weixin_30832405的博客
11-29 245
来源： https://zcopy.wordpress.com/ 说明： 本文不是对原文的逐字逐句翻译，而是摘取核心部分以介绍RDMA Send操作(后面凡是提到RDMA send, 都对应于IBA里的send操作)。文中给出的例子非常浅显易懂，很值得一读。 1. What is RDMA | 什么是RDMA RDMA is Remote Direct Memory Acces...
【verbs】IBV_WR API(3) Libibverbs Programmer’s Manual
bandaoyu的note
12-11 982
阅读源码的时候，发现一些陌生 api，如ibv_wr_send。搜索后才发现是一些dev的api /testing/libibverbs-dev/ ibv_wr_send(3) 源文：ibv_wr_send(3) — libibverbs-dev — Debian testing — Debian Manpages 描述 verbs API (ibv_wr_*) 允许使用函数调用而不是基于结构的 ibv_post_send() 方案将工作高效地发布到发送队列。这种方法旨在最大限度...
【RDMA】RDMA编程实例（IBV Verbs ）
小鱼菜鸟的博客
04-22 1639
目录 更多例子： 基于Verbs的RDMA RC通信编程示例概要 Main() { print_config() resources_init() resources_create() sock_connect() connect_qp() post_send poll_completion resources_destroy ...
【vbers】ibv_post_send|IBV_SEND_SOLICITED|RDMA
bandaoyu的note
01-27 2268
ibv_post_send() 函数原型为 int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr); 其中struct ibv_send_wr结构体的定义为： struct ibv_send_wr { uint64_t wr_id; struct ibv_send_wr *next; ..
rdma-core依赖下载
最新发布
12-27
rdma-core是用于在远程直接内存访问（RDMA）技术上执行的用户态库。要下载rdma-core的依赖，首先需要检查系统上是否已经安装了必要的依赖，例如cmake、libudev、libnl等。可以使用系统包管理器（如apt、yum等）来安装这些依赖。 一旦依赖项安装完成，可以从rdma-core的官方GitHub仓库中克隆代码，然后使用cmake来构建和编译rdma-core。在构建过程中，cmake会自动检测系统上是否安装了必要的依赖，并根据需要进行配置。 在构建和编译完成后，可以使用make install来将rdma-core安装到系统中。这样，rdma-core的依赖就会被自动安装到系统的标准位置，可以被其他程序所引用和使用。 除了从源代码构建和安装外，也可以通过系统包管理器直接安装预编译的rdma-core包。这样可以更加方便地获取rdma-core及其依赖项，同时也能够通过包管理器来进行更新和卸载操作。 总之，下载rdma-core的依赖需要首先安装必要的依赖项，然后通过源代码构建或者包管理器安装rdma-core。这样就可以在系统中使用rdma-core提供的RDMA功能了。

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值