server.c
#include <stdio.h>
#include <netdb.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>
const int BUFFER_SIZE = 1024;
struct context {
struct ibv_context *ctx;
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_comp_channel *comp_channel;
pthread_t cq_poller_thread;
};
static struct context *s_ctx = NULL;
struct connection {
struct ibv_qp *qp;
struct ibv_mr *recv_mr;
struct ibv_mr *send_mr;
char *recv_region;
char *send_region;
struct rdma_cm_id *id;
int num_completions;
};
void *
poll_cq(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx);
ibv_ack_cq_events(cq, 1);
ibv_req_notify_cq(cq, 0);
while (ibv_poll_cq(cq, 1, &wc)) {
struct connection *conn = (struct connection *)(uintptr_t)(wc.wr_id);
if (wc.status != IBV_WC_SUCCESS) {
printf("on_completion: status is not IBV_WC_SUCCESS.");
}
if (wc.opcode & IBV_WC_RECV) {
printf("received message: %s\n", conn->recv_region);
} else if (wc.opcode == IBV_WC_SEND) {
printf("send completed successfully.\n");
}
/*这个在客户端调用了,服务端就不用了*/
/*if (++conn->num_completions == 2){
rdma_disconnect(conn->id);
printf("server disconnect\n");
}*/
}
}
return NULL;
}
int connect_request(struct rdma_cm_id *id)
{
struct ibv_qp_init_attr qp_attr;
struct rdma_conn_param cm_params;
struct ibv_recv_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
struct connection *conn;
if (s_ctx) {
if (s_ctx->ctx != id->verbs)
printf("cannot handle events in more than one context.");
return -1;
}
s_ctx = (struct context *)malloc(sizeof(struct context));
if(!s_ctx){
printf("malloc error\n");
return -1;
}
s_ctx->ctx = id->verbs;
s_ctx->pd = ibv_alloc_pd(s_ctx->ctx);
if(!s_ctx->pd){
printf("ibv_alloc_pd error\n");
}
s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx);
if(!s_ctx->comp_channel){
printf("ibv_create_comp_channel error\n");
}
s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0);
if(!s_ctx->cq){
printf("ibv_create_cq error\n");
}
if(ibv_req_notify_cq(s_ctx->cq, 0)){
printf("ibv_req_notify_cq error\n");
}
pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL);
memset(&qp_attr, 0, sizeof(qp_attr));
qp_attr.send_cq = s_ctx->cq;
qp_attr.recv_cq = s_ctx->cq;
qp_attr.qp_type = IBV_QPT_RC;
qp_attr.cap.max_send_wr = 10;
qp_attr.cap.max_recv_wr = 10;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_sge = 1;
if(rdma_create_qp(id, s_ctx->pd, &qp_attr)) {
printf("rdma_create_qp error\n");
}
id->context = conn = (struct connection *)malloc(sizeof(struct connection));
conn->qp = id->qp;
conn->id = id;
conn->num_completions = 0;
conn->send_region = malloc(BUFFER_SIZE);
conn->recv_region = malloc(BUFFER_SIZE);
conn->send_mr = ibv_reg_mr(s_ctx->pd, conn->send_region, BUFFER_SIZE, 0);
if(!conn->send_mr) {
printf("ibv_reg_mr error\n");
}
conn->recv_mr = ibv_reg_mr(s_ctx->pd, conn->recv_region, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE);
if(!conn->recv_mr){
printf("ibv_reg_mr error\n");
}
sge.addr = (uintptr_t)conn->recv_region;
sge.length = BUFFER_SIZE;
sge.lkey = conn->recv_mr->lkey;
wr.wr_id = (uintptr_t)conn;
wr.next = NULL;
wr.sg_list = &sge;
wr.num_sge = 1;
if(ibv_post_recv(conn->qp, &wr, &bad_wr)) {
printf("ibv_post_recv error\n");
}
memset(&cm_params, 0, sizeof(cm_params));
/*走到这里,应该就会发起建立连接产生的第二个包,客户端收到这个包以后
应该会产生一个RDMA_CM_EVENT_ESTABLISHED事件,然后回CM ReadyToUse这个包,
服务段应该就会收到RDMA_CM_EVENT_ESTABLISHED事件?只是猜测*/
if(rdma_accept(id, &cm_params)) {
printf("rdma_accept error\n");
}
return 0;
}
int
connect_event(void *context)
{
struct connection *conn = (struct connection *)context;
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
snprintf(conn->send_region, BUFFER_SIZE, "message from passive/server side with pid %d", getpid());
memset(&wr, 0, sizeof(wr));
sge.addr = (uintptr_t)conn->send_region;
sge.length = BUFFER_SIZE;
sge.lkey = conn->send_mr->lkey;
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_SEND;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
ibv_post_send(conn->qp, &wr, &bad_wr);
return 0;
}
int
disconnect_event(struct rdma_cm_id *id)
{
struct connection *conn = (struct connection *)id->context;
rdma_destroy_qp(id);
ibv_dereg_mr(conn->send_mr);
ibv_dereg_mr(conn->recv_mr);
free(conn->send_region);
free(conn->recv_region);
free(conn);
rdma_destroy_id(id);
return 0;
}
int
server_event_loop(struct rdma_cm_event *event)
{
int ret = 0;
/*客户端发起连接以后就会收到这个事件*/
if(event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
ret = connect_request(event->id);
printf("RDMA_CM_EVENT_CONNECT_REQUEST\n");
} else if(event->event == RDMA_CM_EVENT_ESTABLISHED) {
/*服务端可以走到这里应该建立连接的过程已经ok了*/
ret = connect_event(event->id->context);
printf("RDMA_CM_EVENT_ESTABLISHED \n");
} else if(event->event == RDMA_CM_EVENT_DISCONNECTED){
ret = disconnect_event(event->id);
printf("RDMA_CM_EVENT_DISCONNECTED \n");
} else {
printf("unknown event\n");
}
return ret;
}
int main(int argc, char **argv)
{
struct sockaddr_in addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *listener = NULL;
struct rdma_event_channel *ec = NULL;
uint16_t port = 0, ret = 0;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
ec = rdma_create_event_channel();
if(!ec) {
printf("rdma_create_event_channel error\n");
return -1;
}
ret = rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP);
if(ret) {
printf("rdma_create_id error\n");
return -1;
}
ret = rdma_bind_addr(listener, (struct sockaddr *)&addr);
if(ret) {
printf("rdma_bind_addr error\n");
return -1;
}
ret = rdma_listen(listener, 10);
if(ret) {
printf("rdma_listen error\n");
return -1;
}
port = ntohs(rdma_get_src_port(listener));
printf("listening on port %d.\n", port);
/*阻塞调用*/
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (server_event_loop(&event_copy)) {
break;
}
}
rdma_destroy_id(listener);
rdma_destroy_event_channel(ec);
return 0;
}
client.c
#include <stdio.h>
#include <netdb.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>
const int BUFFER_SIZE = 1024;
const int TIMEOUT_IN_MS = 500;
struct context {
struct ibv_context *ctx;
struct ibv_pd *pd;
struct ibv_cq *cq;
struct ibv_comp_channel *comp_channel;
pthread_t cq_poller_thread;
};
static struct context *s_ctx = NULL;
struct connection {
struct rdma_cm_id *id;
struct ibv_qp *qp;
struct ibv_mr *recv_mr;
struct ibv_mr *send_mr;
char *recv_region;
char *send_region;
int num_completions;
};
void *
poll_cq(void *ctx)
{
struct ibv_cq *cq;
struct ibv_wc wc;
while (1) {
ibv_get_cq_event(s_ctx->comp_channel, &cq, &ctx);
ibv_ack_cq_events(cq, 1);
ibv_req_notify_cq(cq, 0);
while (ibv_poll_cq(cq, 1, &wc)) {
struct connection *conn = (struct connection *)(uintptr_t)(wc.wr_id);
if (wc.status != IBV_WC_SUCCESS)
printf("on_completion: status is not IBV_WC_SUCCESS.");
if (wc.opcode & IBV_WC_RECV)
printf("received message: %s\n", conn->recv_region);
else if (wc.opcode == IBV_WC_SEND)
printf("send completed successfully.\n");
else
printf("poll cq wc status error");
if (++conn->num_completions == 2)
rdma_disconnect(conn->id);
}
}
return NULL;
}
int
addr_resolved(struct rdma_cm_id *id)
{
struct ibv_qp_init_attr qp_attr;
struct ibv_recv_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
struct connection *conn;
if (s_ctx) {
if (s_ctx->ctx != id->verbs)
printf("cannot handle events in more than one context.");
return -1;
}
s_ctx = (struct context *)malloc(sizeof(struct context));
if(!s_ctx){
printf("malloc error\n");
return -1;
}
s_ctx->ctx = id->verbs;
s_ctx->pd = ibv_alloc_pd(s_ctx->ctx);
if(!s_ctx->ctx){
printf("ibv_alloc_pd error\n");
}
s_ctx->comp_channel = ibv_create_comp_channel(s_ctx->ctx);
if(!s_ctx->comp_channel){
printf("ibv_create_comp_channel error\n");
}
s_ctx->cq = ibv_create_cq(s_ctx->ctx, 10, NULL, s_ctx->comp_channel, 0);
if(!s_ctx->cq){
printf("ibv_create_cq error\n");
}
if(ibv_req_notify_cq(s_ctx->cq, 0)){
printf("ibv_req_notify_cq error\n");
}
pthread_create(&s_ctx->cq_poller_thread, NULL, poll_cq, NULL);
memset(&qp_attr, 0, sizeof(qp_attr));
qp_attr.send_cq = s_ctx->cq;
qp_attr.recv_cq = s_ctx->cq;
qp_attr.qp_type = IBV_QPT_RC;
qp_attr.cap.max_send_wr = 10;
qp_attr.cap.max_recv_wr = 10;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_sge = 1;
if(rdma_create_qp(id, s_ctx->pd, &qp_attr)) {
printf("rdma_create_qp error\n");
}
conn = (struct connection *)malloc(sizeof(struct connection));
id->context = conn;
conn->id = id;
conn->qp = id->qp;
conn->num_completions = 0;
conn->send_region = malloc(BUFFER_SIZE);
conn->recv_region = malloc(BUFFER_SIZE);
conn->send_mr = ibv_reg_mr(s_ctx->pd, conn->send_region, BUFFER_SIZE, 0);
if(!conn->send_mr) {
printf("ibv_reg_mr error\n");
}
conn->recv_mr = ibv_reg_mr(s_ctx->pd, conn->recv_region, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE);
if(!conn->recv_mr){
printf("ibv_reg_mr error\n");
}
wr.wr_id = (uintptr_t)conn;
wr.next = NULL;
wr.sg_list = &sge;
wr.num_sge = 1;
sge.addr = (uintptr_t)conn->recv_region;
sge.length = BUFFER_SIZE;
sge.lkey = conn->recv_mr->lkey;
/*异步的过程*/
if(ibv_post_recv(conn->qp, &wr, &bad_wr)) {
printf("ibv_post_recv error\n");
}
/*调用以后生成RDMA_CM_EVENT_ROUTE_RESOLVED事件*/
if(rdma_resolve_route(id, TIMEOUT_IN_MS)) {
printf("rdma_resolve_route error\n");
}
return 0;
}
int
route_resolved(struct rdma_cm_id *id)
{
struct rdma_conn_param cm_params;
memset(&cm_params, 0, sizeof(cm_params));
/*对服务端发起连接,这个id应该就包含了要互相交互的信息,
调用以后在服务端调用rdma_accept以后应该就会产生RDMA_CM_EVENT_ESTABLISHED事件,
具体要看源码分析*/
rdma_connect(id, &cm_params);
return 0;
}
int
connection(void *context)
{
struct connection *conn = (struct connection *)context;
struct ibv_send_wr wr, *bad_wr = NULL;
struct ibv_sge sge;
snprintf(conn->send_region, BUFFER_SIZE, "message from active/client side with pid %d", getpid());
memset(&wr, 0, sizeof(wr));
sge.addr = (uintptr_t)conn->send_region;
sge.length = BUFFER_SIZE;
sge.lkey = conn->send_mr->lkey;
wr.wr_id = (uintptr_t)conn;
wr.opcode = IBV_WR_SEND;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = IBV_SEND_SIGNALED;
ibv_post_send(conn->qp, &wr, &bad_wr);
return 0;
}
int
disconnect(struct rdma_cm_id *id)
{
struct connection *conn = (struct connection *)id->context;
rdma_destroy_qp(id);
ibv_dereg_mr(conn->send_mr);
ibv_dereg_mr(conn->recv_mr);
free(conn->send_region);
free(conn->recv_region);
free(conn);
rdma_destroy_id(id);
return 0;
}
int
client_event_loop(struct rdma_cm_event *event)
{
int ret = 0;
if (event->event == RDMA_CM_EVENT_ADDR_RESOLVED) {
ret = addr_resolved(event->id);
printf("RDMA_CM_EVENT_ADDR_RESOLVED\n");
} else if (event->event == RDMA_CM_EVENT_ROUTE_RESOLVED) {
ret = route_resolved(event->id);
printf("RDMA_CM_EVENT_ROUTE_RESOLVED\n");
} else if (event->event == RDMA_CM_EVENT_ESTABLISHED) {
/*客户端可以走到这里应该建立连接的过程已经ok了*/
ret = connection(event->id->context);
printf("RDMA_CM_EVENT_ESTABLISHED\n");
} else if (event->event == RDMA_CM_EVENT_DISCONNECTED){
ret = disconnect(event->id);
printf("RDMA_CM_EVENT_DISCONNECTED\n");
} else{
printf("unknown event\n");
}
return ret;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn = NULL;
struct rdma_event_channel *ec = NULL;
if(argc != 3)
printf("usage: client <server-address> <server-port>\n");
if(getaddrinfo(argv[1], argv[2], NULL, &addr))
return -1;
ec = rdma_create_event_channel();
if(!ec) {
printf("rdma_create_event_channel error\n");
return -1;
}
if(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)) {
printf("rdma_create_id error\n");
return -1;
}
if(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)) {
printf("rdma_resolve_addr error\n");
return -1;
}
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (client_event_loop(&event_copy)) {
break;
}
}
rdma_destroy_event_channel(ec);
return 0;
}
Makefile
.PHONY: clean
CFLAGS := -Wall -Werror -g
LD := gcc
LDLIBS := -lrdmacm -libverbs -lpthread
APPS := client server
all: ${APPS}
clean:
rm -f *.o ${APPS}
客户端与服务端互相发送一次消息给对方。
RDMA编程涉及的API比较多,可以抓包去分析这个过程。