rdma-core ud_pingpong.c阅读记录

已于 2022-11-07 19:46:39 修改
阅读量639 收藏
点赞数
分类专栏： RDMA 文章标签： c语言 开发语言 linux
于 2022-11-07 19:45:01 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_38158479/article/details/127737631
版权 
#define _GNU_SOURCE
#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>

#include "pingpong.h"

enum {
	PINGPONG_RECV_WRID = 1,
	PINGPONG_SEND_WRID = 2,
};

static int page_size;
static int validate_buf;

struct pingpong_context {
	struct ibv_context	*context;
	struct ibv_comp_channel *channel;
	struct ibv_pd		*pd;
	struct ibv_mr		*mr;
	struct ibv_cq		*cq;
	struct ibv_qp		*qp;
	struct ibv_ah		*ah;
	char			*buf;
	int			 size;
	int			 send_flags;
	int			 rx_depth;
	int			 pending;
	struct ibv_port_attr     portinfo;
};

struct pingpong_dest {
	int lid;
	int qpn;
	int psn;
	union ibv_gid gid;
};

static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
			  int sl, struct pingpong_dest *dest, int sgid_idx)
{
	struct ibv_ah_attr ah_attr = {
		.is_global     = 0,
		.dlid          = dest->lid,
		.sl            = sl,
		.src_path_bits = 0,
		.port_num      = port
	};
	struct ibv_qp_attr attr = {
		.qp_state		= IBV_QPS_RTR
	};

	if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
		fprintf(stderr, "Failed to modify QP to RTR\n");
		return 1;
	}

	attr.qp_state	= IBV_QPS_RTS;
	attr.sq_psn	    = my_psn;

	if (ibv_modify_qp(ctx->qp, &attr,
			  IBV_QP_STATE              |
			  IBV_QP_SQ_PSN)) {
		fprintf(stderr, "Failed to modify QP to RTS\n");
		return 1;
	}

	if (dest->gid.global.interface_id) {
		ah_attr.is_global = 1;
		ah_attr.grh.hop_limit = 1;
		ah_attr.grh.dgid = dest->gid;
		ah_attr.grh.sgid_index = sgid_idx;
	}

	ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
	if (!ctx->ah) {
		fprintf(stderr, "Failed to create AH\n");
		return 1;
	}

	return 0;
}

static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
						 const struct pingpong_dest *my_dest)
{
	struct addrinfo *res, *t;
	struct addrinfo hints = {
		.ai_family   = AF_UNSPEC,
		.ai_socktype = SOCK_STREAM
	};
	char *service;
	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
	int n;
	int sockfd = -1;
	struct pingpong_dest *rem_dest = NULL;
	char gid[33];

	if (asprintf(&service, "%d", port) < 0)
		return NULL;

	n = getaddrinfo(servername, service, &hints, &res);

	if (n < 0) {
		fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
		free(service);
		return NULL;
	}

	for (t = res; t; t = t->ai_next) {
		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
		if (sockfd >= 0) {
			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
				break;
			close(sockfd);
			sockfd = -1;
		}
	}

	freeaddrinfo(res);
	free(service);

	if (sockfd < 0) {
		fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
		return NULL;
	}

	gid_to_wire_gid(&my_dest->gid, gid);
	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid);					
	if (write(sockfd, msg, sizeof msg) != sizeof msg) {
		fprintf(stderr, "Couldn't send local address\n");
		goto out;
	}

	if (read(sockfd, msg, sizeof msg) != sizeof msg ||
	    write(sockfd, "done", sizeof "done") != sizeof "done") {
		perror("client read/write");
		fprintf(stderr, "Couldn't read/write remote address\n");
		goto out;
	}

	rem_dest = malloc(sizeof *rem_dest);
	if (!rem_dest)
		goto out;

	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);						
	wire_gid_to_gid(gid, &rem_dest->gid);
out:
	close(sockfd);
	return rem_dest;
}

static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
						 int ib_port, int port, int sl,
						 const struct pingpong_dest *my_dest,
						 int sgid_idx)
{
	struct addrinfo *res, *t;
	struct addrinfo hints = {
		.ai_flags    = AI_PASSIVE,
		.ai_family   = AF_UNSPEC,
		.ai_socktype = SOCK_STREAM
	};
	char *service;
	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
	int n;
	int sockfd = -1, connfd;
	struct pingpong_dest *rem_dest = NULL;
	char gid[33];

	if (asprintf(&service, "%d", port) < 0)
		return NULL;

	n = getaddrinfo(NULL, service, &hints, &res);

	if (n < 0) {
		fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
		free(service);
		return NULL;
	}

	for (t = res; t; t = t->ai_next) {
		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
		if (sockfd >= 0) {
			n = 1;

			setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);

			if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
				break;
			close(sockfd);
			sockfd = -1;
		}
	}

	freeaddrinfo(res);
	free(service);

	if (sockfd < 0) {
		fprintf(stderr, "Couldn't listen to port %d\n", port);
		return NULL;
	}

	listen(sockfd, 1);
	connfd = accept(sockfd, NULL, NULL);
	close(sockfd);
	if (connfd < 0) {
		fprintf(stderr, "accept() failed\n");
		return NULL;
	}

	n = read(connfd, msg, sizeof msg);
	if (n != sizeof msg) {
		perror("server read");
		fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
		goto out;
	}

	rem_dest = malloc(sizeof *rem_dest);
	if (!rem_dest)
		goto out;

    /*从msg里面读取数据，按照一定格式写入rem_dest的各个变量里*/
	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
							&rem_dest->psn, gid);
    /*把gid的值写入rem_dest->gid*/
	wire_gid_to_gid(gid, &rem_dest->gid);

	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
								sgid_idx)) {
		fprintf(stderr, "Couldn't connect to remote QP\n");
		free(rem_dest);
		rem_dest = NULL;
		goto out;
	}

	gid_to_wire_gid(&my_dest->gid, gid);
	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
							my_dest->psn, gid);
	if (write(connfd, msg, sizeof msg) != sizeof msg ||
	    read(connfd, msg, sizeof msg) != sizeof "done") {
		fprintf(stderr, "Couldn't send/recv local address\n");
		free(rem_dest);
		rem_dest = NULL;
		goto out;
	}
out:
	close(connfd);
	return rem_dest;
}

static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
					    int rx_depth, int port,
					    int use_event)
{
	struct pingpong_context *ctx;
    /*申请内存--pingpong_context*/
	ctx = malloc(sizeof *ctx);
	if (!ctx)
		return NULL;

	ctx->size       = size;//1024
	ctx->send_flags = IBV_SEND_SIGNALED;
	ctx->rx_depth   = rx_depth;
    /*申请内存块-大小为size+40，以page_size为边界对齐*/
	ctx->buf = memalign(page_size, size + 40);
	if (!ctx->buf) {
		fprintf(stderr, "Couldn't allocate work buf.\n");
		goto clean_ctx;
	}

	/* FIXME memset(ctx->buf, 0, size + 40); */
	memset(ctx->buf, 0x7b, size + 40);

	ctx->context = ibv_open_device(ib_dev);
	if (!ctx->context) {
		fprintf(stderr, "Couldn't get context for %s\n",
			ibv_get_device_name(ib_dev));
		goto clean_buffer;
	}

	{
		struct ibv_port_attr port_info = {};
		int mtu;

		if (ibv_query_port(ctx->context, port, &port_info)) {
			fprintf(stderr, "Unable to query port info for port %d\n", port);
			goto clean_device;
		}
		mtu = 1 << (port_info.active_mtu + 7);
		if (size > mtu) {
			fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
			goto clean_device;
		}
	}
    /*如果需要用事件通知就进来，如果不用就直接调用ibv_poll_cq*/
	if (use_event) {
		ctx->channel = ibv_create_comp_channel(ctx->context);
		if (!ctx->channel) {
			fprintf(stderr, "Couldn't create completion channel\n");
			goto clean_device;
		}
	} else
		ctx->channel = NULL;

	ctx->pd = ibv_alloc_pd(ctx->context);
	if (!ctx->pd) {
		fprintf(stderr, "Couldn't allocate PD\n");
		goto clean_comp_channel;
	}

	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
	if (!ctx->mr) {
		fprintf(stderr, "Couldn't register MR\n");
		goto clean_pd;
	}

	ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
				ctx->channel, 0);
	if (!ctx->cq) {
		fprintf(stderr, "Couldn't create CQ\n");
		goto clean_mr;
	}

	{
		struct ibv_qp_attr attr;
		struct ibv_qp_init_attr init_attr = {
			.send_cq = ctx->cq,
			.recv_cq = ctx->cq,
			.cap = {
				.max_send_wr  = 1,
				.max_recv_wr  = rx_depth,/*在没有收到相应的poll_cq事件时，一次性下发post_recv的次数*/
				.max_send_sge = 1,
			.max_recv_sge = 100
			},
			.qp_type = IBV_QPT_UD,/*指明是UD操作*/
		};

		ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
		if (!ctx->qp)  {
			fprintf(stderr, "Couldn't create QP\n");
			goto clean_cq;
		}

		ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
		if (init_attr.cap.max_inline_data >= size) {
			ctx->send_flags |= IBV_SEND_INLINE;
		}
	}

	{/*修改QP状态*/
		struct ibv_qp_attr attr = {
			.qp_state        = IBV_QPS_INIT,
			.pkey_index      = 0,
			.port_num        = port,
			.qkey            = 0x11111111
		};

		if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
				  IBV_QP_PORT | IBV_QP_QKEY)) {	  
			fprintf(stderr, "Failed to modify QP to INIT\n");
			goto clean_qp;
		}
	}
	return ctx;
clean_qp:
	ibv_destroy_qp(ctx->qp);
clean_cq:
	ibv_destroy_cq(ctx->cq);
clean_mr:
	ibv_dereg_mr(ctx->mr);
clean_pd:
	ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
	if (ctx->channel)
		ibv_destroy_comp_channel(ctx->channel);
clean_device:
	ibv_close_device(ctx->context);
clean_buffer:
	free(ctx->buf);
clean_ctx:
	free(ctx);
	return NULL;
}

static int pp_close_ctx(struct pingpong_context *ctx)
{
	if (ibv_destroy_qp(ctx->qp)) {
		fprintf(stderr, "Couldn't destroy QP\n");
		return 1;
	}

	if (ibv_destroy_cq(ctx->cq)) {
		fprintf(stderr, "Couldn't destroy CQ\n");
		return 1;
	}

	if (ibv_dereg_mr(ctx->mr)) {
		fprintf(stderr, "Couldn't deregister MR\n");
		return 1;
	}

	if (ibv_destroy_ah(ctx->ah)) {
		fprintf(stderr, "Couldn't destroy AH\n");
		return 1;
	}

	if (ibv_dealloc_pd(ctx->pd)) {
		fprintf(stderr, "Couldn't deallocate PD\n");
		return 1;
	}

	if (ctx->channel) {
		if (ibv_destroy_comp_channel(ctx->channel)) {
			fprintf(stderr, "Couldn't destroy completion channel\n");
			return 1;
		}
	}

	if (ibv_close_device(ctx->context)) {
		fprintf(stderr, "Couldn't release context\n");
		return 1;
	}
	free(ctx->buf);
	free(ctx);
	return 0;
}

static int pp_post_recv(struct pingpong_context *ctx, int n)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size + 40,/*要额外加40个字节*/
		.lkey	= ctx->mr->lkey
	};
	struct ibv_recv_wr wr = {
		.wr_id	    = PINGPONG_RECV_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
	};
	struct ibv_recv_wr *bad_wr;
	int i;
	for (i = 0; i < n; ++i)
		if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
			break;

	return i;
}

static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf + 40,
		.length = ctx->size,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_send_wr wr = {
		.wr_id	    = PINGPONG_SEND_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
		.opcode     = IBV_WR_SEND,
		.send_flags = ctx->send_flags,
		.wr         = {
		    .ud = {/*UD请求，将WR与AH绑定*/
				 .ah          = ctx->ah,
				 .remote_qpn  = qpn,
				 .remote_qkey = 0x11111111
			 }
		}
	};
	struct ibv_send_wr *bad_wr;
	return ibv_post_send(ctx->qp, &wr, &bad_wr);
}

static void usage(const char *argv0)
{
	printf("Usage:\n");
	printf("  %s            start a server and wait for connection\n", argv0);
	printf("  %s <host>     connect to server at <host>\n", argv0);
	printf("\n");
	printf("Options:\n");
	printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
	printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
	printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
	printf("  -s, --size=<size>      size of message to exchange (default 2048)\n");
	printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
	printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
    printf("  -l, --sl=<SL>          send messages with service level <SL> (default 0)\n");
	printf("  -e, --events           sleep on CQ events (default poll)\n");
	printf("  -g, --gid-idx=<gid index> local port gid index\n");
	printf("  -c, --chk              validate received buffer\n");
}

int main(int argc, char *argv[])
{
	struct ibv_device      **dev_list;
	struct ibv_device	   *ib_dev;
	struct pingpong_context *ctx;
	struct pingpong_dest     my_dest;
	struct pingpong_dest    *rem_dest;
	struct timeval           start, end;
	char                    *ib_devname = NULL;
	char                    *servername = NULL;
	unsigned int             port = 18515;
	int                      ib_port = 1;
	unsigned int             size = 1024;
	unsigned int             rx_depth = 500;
	unsigned int             iters = 1000;
	int                      use_event = 0;
	int                      routs;
	int                      rcnt, scnt;
	int                      num_cq_events = 0;
	int                      sl = 0;
	int			             gidx = -1;
	char			         gid[33];

	srand48(getpid() * time(NULL));
	while (1) {
		int c;
		static struct option long_options[] = {
			{ .name = "port",     .has_arg = 1, .val = 'p' },
			{ .name = "ib-dev",   .has_arg = 1, .val = 'd' },
			{ .name = "ib-port",  .has_arg = 1, .val = 'i' },
			{ .name = "size",     .has_arg = 1, .val = 's' },
			{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
			{ .name = "iters",    .has_arg = 1, .val = 'n' },
			{ .name = "sl",       .has_arg = 1, .val = 'l' },
			{ .name = "events",   .has_arg = 0, .val = 'e' },
			{ .name = "gid-idx",  .has_arg = 1, .val = 'g' },
			{ .name = "chk",      .has_arg = 0, .val = 'c' },
			{}
		};

		c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:c", long_options,NULL);	
		if (c == -1)
			break;

		switch (c) {
		case 'p':
			port = strtol(optarg, NULL, 0);
			if (port > 65535) {
				usage(argv[0]);
				return 1;
			}
			break;
		case 'd':
			ib_devname = strdupa(optarg);
			break;
		case 'i':
			ib_port = strtol(optarg, NULL, 0);
			if (ib_port < 1) {
				usage(argv[0]);
				return 1;
			}
			break;
		case 's':
			size = strtoul(optarg, NULL, 0);
			break;
		case 'r':
			rx_depth = strtoul(optarg, NULL, 0);
			break;
		case 'n':
			iters = strtoul(optarg, NULL, 0);
			break;
		case 'l':
			sl = strtol(optarg, NULL, 0);
			break;
		case 'e':
			++use_event;
			break;
		case 'g':
			gidx = strtol(optarg, NULL, 0);
			break;
		case 'c':
			validate_buf = 1;
			break;
		default:
			usage(argv[0]);
			return 1;
		}
	}

    /*获取IP*/
	if (optind == argc - 1)
		servername = strdupa(argv[optind]);
	else if (optind < argc) {
		usage(argv[0]);
		return 1;
	}
    /*打印出来就是4096*/
	page_size = sysconf(_SC_PAGESIZE);
    /*获取设备列表*/
	dev_list = ibv_get_device_list(NULL);
	if (!dev_list) {
		perror("Failed to get IB devices list");
		return 1;
	}
    /*如果没有指定使用哪一个设备，就默认使用第一个*/
	if (!ib_devname) {
		ib_dev = *dev_list;
		if (!ib_dev) {
			fprintf(stderr, "No IB devices found\n");
			return 1;
		}
	} else {/*指定使用哪一个设备*/
		int i;
		for (i = 0; dev_list[i]; ++i)
			if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
				break;
		ib_dev = dev_list[i];
		if (!ib_dev) {
			fprintf(stderr, "IB device %s not found\n", ib_devname);
			return 1;
		}
	}

	ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
	if (!ctx)
		return 1;
	routs = pp_post_recv(ctx, ctx->rx_depth);
	if (routs < ctx->rx_depth) {
		fprintf(stderr, "Couldn't post receive (%d)\n", routs);
		return 1;
	}

	if (use_event)
		if (ibv_req_notify_cq(ctx->cq, 0)) {
			fprintf(stderr, "Couldn't request CQ notification\n");
			return 1;
		}

	if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
		fprintf(stderr, "Couldn't get port info\n");
		return 1;
	}
	my_dest.lid = ctx->portinfo.lid;
	my_dest.qpn = ctx->qp->qp_num;
	my_dest.psn = lrand48() & 0xffffff;
	if (gidx >= 0) {
		if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
			fprintf(stderr, "Could not get local gid for gid index "
								"%d\n", gidx);
			return 1;
		}
	} else
		memset(&my_dest.gid, 0, sizeof my_dest.gid);

	inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
	printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
	       my_dest.lid, my_dest.qpn, my_dest.psn, gid);

	if (servername)
		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
	else
		rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx);
							
	if (!rem_dest)
		return 1;

	inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
	printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
	       rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);

	if (servername)
		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx))							
			return 1;

	ctx->pending = PINGPONG_RECV_WRID;
	if (servername) {
		if (validate_buf)
			for (int i = 0; i < size; i += page_size)
                /*额外添加40字节保存GRH*/
				ctx->buf[i + 40] = i / page_size % sizeof(char);
        /*客户端发送-->服务端*/
		if (pp_post_send(ctx, rem_dest->qpn)) {
			fprintf(stderr, "Couldn't post send\n");
			return 1;
		}
		ctx->pending |= PINGPONG_SEND_WRID;
	}

	if (gettimeofday(&start, NULL)) {
		perror("gettimeofday");
		return 1;
	}

	rcnt = scnt = 0;
	while (rcnt < iters || scnt < iters) {
        /*是否使用事件通知*/
		if (use_event) {
			struct ibv_cq *ev_cq;
			void          *ev_ctx;
            /*阻塞等待响应事件的发生*/
			if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
				fprintf(stderr, "Failed to get cq_event\n");
				return 1;
			}
            /*响应事件累加*/
			++num_cq_events;
			if (ev_cq != ctx->cq) {
				fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
				return 1;
			}
            /*来了一次响应事件以后，这个要在调用一次*/
			if (ibv_req_notify_cq(ctx->cq, 0)) {
				fprintf(stderr, "Couldn't request CQ notification\n");
				return 1;
			}
		}

		{
			struct ibv_wc wc[2];
			int ne, i;
			do {
				ne = ibv_poll_cq(ctx->cq, 2, wc);/*非阻塞*/
				if (ne < 0) {
					fprintf(stderr, "poll CQ failed %d\n", ne);
					return 1;
				}
			} while (!use_event && ne < 1);

			for (i = 0; i < ne; ++i) {
				if (wc[i].status != IBV_WC_SUCCESS) {
					fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
						ibv_wc_status_str(wc[i].status), wc[i].status, (int) wc[i].wr_id);	
					return 1;
				}
				switch ((int) wc[i].wr_id) {
				case PINGPONG_SEND_WRID:
					++scnt;
					break;
				case PINGPONG_RECV_WRID:
					if (--routs <= 1) {
						routs += pp_post_recv(ctx, ctx->rx_depth - routs);
						if (routs < ctx->rx_depth) {
							fprintf(stderr, "Couldn't post receive (%d)\n", routs);	
							return 1;
						}
					}
					++rcnt;
					break;
				default:
					fprintf(stderr, "Completion for unknown wr_id %d\n", (int) wc[i].wr_id);
					return 1;
				}

				ctx->pending &= ~(int) wc[i].wr_id;
				if (scnt < iters && !ctx->pending) {
					if (pp_post_send(ctx, rem_dest->qpn)) {
						fprintf(stderr, "Couldn't post send\n");
						return 1;
					}
					ctx->pending = PINGPONG_RECV_WRID | PINGPONG_SEND_WRID;	       
				}
			}
		}
	}
    
	if (gettimeofday(&end, NULL)) {
		perror("gettimeofday");
		return 1;
	}
	{
       /*
       1:Mbit/s = 10^6 bit
       2:MB/s = 10^6 byte
       3:Mbit/s / 8 = MB/s
       */
	    /*总共花费的us数*/
		float usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec);
        /*总共传输和接收的字节数*/
		long long bytes = (long long) size * iters * 2;

        /*Mbit/sec <==> Mbit/usec*/
		printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", bytes, usec / 1000000., bytes * 8. / usec);   
		printf("%d iters in %.2f seconds = %.2f usec/iter\n", iters, usec / 1000000., usec / iters);

		if ((!servername) && (validate_buf)) {
			for (int i = 0; i < size; i += page_size)
				if (ctx->buf[i + 40] != i / page_size % sizeof(char))
					printf("invalid data in page %d\n", i / page_size);		       
		}
	}

	ibv_ack_cq_events(ctx->cq, num_cq_events);
	if (pp_close_ctx(ctx))
		return 1;
	ibv_free_device_list(dev_list);
	free(rem_dest);
	return 0;
}

enum ibv_mtu pp_mtu_to_enum(int mtu)
{
	switch (mtu) {
	case 256:  return IBV_MTU_256;
	case 512:  return IBV_MTU_512;
	case 1024: return IBV_MTU_1024;
	case 2048: return IBV_MTU_2048;
	case 4096: return IBV_MTU_4096;
	default:   return 0;
	}
}

int pp_get_port_info(struct ibv_context *context, int port,
		     struct ibv_port_attr *attr)
{
	return ibv_query_port(context, port, attr);
}

void wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
{
	char tmp[9];
	__be32 v32;
	int i;
	uint32_t tmp_gid[4];

	for (tmp[8] = 0, i = 0; i < 4; ++i) {
		memcpy(tmp, wgid + i * 8, 8);
		sscanf(tmp, "%x", &v32);
		tmp_gid[i] = be32toh(v32);
	}
	memcpy(gid, tmp_gid, sizeof(*gid));
}

void gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
{
	uint32_t tmp_gid[4];
	int i;

	memcpy(tmp_gid, gid, sizeof(tmp_gid));
	for (i = 0; i < 4; ++i)
		sprintf(&wgid[i * 8], "%08x", htobe32(tmp_gid[i]));
}

利用socket建立连接，然后交换LID.QPN等信息。。。
后续直接下发IBV_POST_SEND就可以了。。
需要注意的是，在指定buffer大小时，要多分配40个字节。
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述

酸菜。
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 3
    评论
专栏目录
rdma-core-35.0.tar.gz
03-27
【rdma-core-35.0.tar.gz】这个压缩包文件包含了远程直接内存访问（RDMA）的核心库，版本为35.0。RDMA是一种高效的数据传输技术，它允许网络上的应用程序直接读写远程系统的内存，而无需通过操作系统内核，从而显著...
rdma-core-22.4-6.el7_9.x86_64.rpm
12-10
官方离线安装包，亲测可用
【报错处理】ib_write_bw执行遇到Failed to modify QP 358 to RTR原因与解决办法？
技术札记
05-19 1070
QP可以处于不同的状态，其中RTR（Ready to Receive）是QP状态之一，表示QP已经准备好接收消息。两个IB接口并未链接，但是两个IB接口的服务器是能够互通的时候会出现该问题。更改拓扑，让测试的两个port在IB上能够通。比如直连或者交换机处理。命令: ib_write_bw 1.1.1.1。
【verbs】ibv_query_qp()
bandaoyu的note
04-28 893
原文：http://www.polarhome.com/service/man/AIX/libs/ofed/ofed_ibv_create_qp.htm ibv_query_qp() Contents[hide] 1Description 2Parameters 3Return Values 4Examples 5FAQs 5.0.0.1Calling every time to ibv_query_qp() when I need a QP attribu...
教你在虚拟机上把普通网卡配置成softroce设备来运行rdma-core中的示例程序
最新发布
妄北y
07-30 894
在虚拟机环境中，将普通网卡配置为 SoftRoCE 设备可以帮助你在没有实际硬件 RDMA 设备的情况下测试和体验 RDMA 功能。SoftRoCE 是 RoCE（RDMA over Converged Ethernet）协议的一个软件实现。
【RDMA】RDMA编程入门--编辑中
bandaoyu的note
07-08 4777
作者：bandaoyu 本文随时更新，地址：首先应该先了解RDMA：https://blog.csdn.net/bandaoyu/article/details/112859853 RDMA 学习资料总目录:https://blog.csdn.net/bandaoyu/article/details/120485737本文所讲述的主要是IBoE的RDMA编程，主要是ROCE、iWrap，与IB可能略有不同。关键词：点对点通信、QP（SQ+RQ）、CQ、*QE RDMA提供了基于消息队列的点对点通信，每个应用
rdma-core/ud_pingpong.c阅读记录
qq_38158479的博客
11-16 497
rdma-core/ud_pingpong.c阅读记录
使用Soft-RoCE实践RDMA
wq897387的专栏
07-23 1126
使用Soft-RoCE实践RDMA
9. RDMA之Queue Pair
weixin_39094034的博客
10-30 1153
QPC全称是Queue Pair Context
rdma-core-devel-22.4-5.el7.x86_64.rpm
12-10
官方离线安装包，亲测可用
rdma-core-17.2-3.el7.x86_64.rpm
11-30
离线安装包，亲测可用
rdma-core-22.4-6.el7_9.i686.rpm
12-10
官方离线安装包，亲测可用
Mellanox RoCE卡调试笔记
热门推荐
winux的专栏
09-24 1万+
拿到3块mellanox 的XC3卡，装到3个服务器上，开始RDMA调试。代码都是我自己的，使用rdma_cm库进行开发。Mellanox的RDMA是基于ROCE协议的，因此需要物理层保证可靠传输，不能丢包。为此交换机需要支持DCB， PFC等流控技术。这样子系统一下子就复杂了，为了简化系统，决定不使用交换机，把服务器之间用光纤直连。好在我只有3台服务器，刚好可以连起来，1台对2台即可。ubuntu
rdma-ud例程
qq_38158479的博客
11-08 543
rdma-ud例程
22. RDMA之基于Socket API的QP间建链
weixin_39094034的博客
11-01 1131
建链，即建立链接（或者连接）。在RDMA技术中，本端和对端的QP之间在建立连接之后才可以通过RDMA协议进行数据交换。我们可以认为建链是正式进行RDMA通信之前所必须进行的准备工作，在建链的过程中，两个节点间会通过某些方式交换一些用于之后的RDMA通信所必须的参数。
IB程序设计流程
MarsYYP的博客
04-23 895
IB程序设计流程 以ibv_ud_pingpong为例 文章目录IB程序设计流程1. 声明控制结构体1.1全局结构体1.2目标地址结构体2. 主函数执行流程2.1准备工作声明变量获取参数2.2 正式工作2.2.1 获取设备2.2.2 初始化全局结构体(核心)----pp_init_ctx2.2.3 接收队列登记任务WR------pp_post_recv2.2.4 事件---ibv_req_notify_cq2.2.5 获取端口信息----pp_get_port_info2.2.6 获取GID属性2.2.7
【RDMA】9. RDMA之Queue Pair
bandaoyu的note
01-25 2729
原文：https://zhuanlan.zhihu.com/p/195757767 Queue Pair 我们曾经在3. RDMA基本元素一文中简单的介绍了QP的概念，本文将更深入的讲解一些关于QP的细节。 基本概念回顾 首先我们来简单回顾下关于QP的基础知识： 根据IB协议中的描述，QP是硬件和软件之间的一个虚拟接口。QP是队列结构，按顺序存储着软件给硬件下发的任务（WQE），WQE中包含从哪里取出多长的数据，并且发送给哪个目的地等等信息。 QP的概念 每个QP间都是独立的，彼此通.
rdma-core-22.1-3.el7.x86_64
08-31
rdma-core-22.1-3.el7.x86_64是一个软件包的名称。它是为CentOS 7 x86_64架构而设计的。RDMA代表远程直接内存访问，它允许计算机之间直接共享内存，从而提供了高性能和低延迟的数据传输。rdma-core是一个开源的RDMA核心库，提供了与RDMA相关的功能和API。版本号“22.1-3”表示此软件包的版本是22.1的第三个修订版本。软件包中的“el7”指的是该软件包适用于CentOS 7操作系统。最后的“x86_64”表示该软件包是为64位的x86架构而编译的。该软件包可能包含rdma-core库、开发工具、文档和示例等。使用此软件包，开发人员可以在CentOS 7系统上实现和优化RDMA功能，以实现高性能和低延迟的数据传输。
评论 3
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值