Soft-iWARP的安装及验证 —>亲测有效(保持ubuntu版本一致)
一、安装
reference: https://www.cnblogs.com/yu121/p/13358572.html
RDMA需要专门的RDMA网卡或者InfiniBand卡,若没有这些硬件设备则可以使用Soft-iWARP。
1、下载ubuntu-16.04.6-server-amd64.iso镜像,新建一个虚拟机,搭好
2、安装好后查看版本号
uname -sr
Linux 4.4.0-24-generic
3、安装特定版本kernel
wget kernel.ubuntu.com/~kernel-ppa/mainline/v4.8.17/linux-headers-4.8.17-040817_4.8.17-040817.201701090438_all.deb
wget kernel.ubuntu.com/~kernel-ppa/mainline/v4.8.17/linux-headers-4.8.17-040817-generic_4.8.17-040817.201701090438_amd64.deb
wget kernel.ubuntu.com/~kernel-ppa/mainline/v4.8.17/linux-image-4.8.17-040817-generic_4.8.17-040817.201701090438_amd64.deb
4、安装内核、重启系统并再次查看内核版本
dpkg -i linux-headers-4.8*.deb linux-image-4.8*.deb
reboot
5、更换apt源
cp /etc/apt/sources.list /etc/apt/sources.list.bak
vim /etc/apt/sources.list
填充如下内容:
# deb cdrom:[Ubuntu 16.04 LTS _Xenial Xerus_ - Release amd64 (20160420.1)]/ xenial main restricted
deb-src http://archive.ubuntu.com/ubuntu xenial main restricted #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse #Added by software-properties
deb http://archive.canonical.com/ubuntu xenial partner
deb-src http://archive.canonical.com/ubuntu xenial partner
deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse
6、更新源
apt update -y
7、安装相关依赖
apt-get install fakeroot build-essential crash kexec-tools makedumpfile kernel-wedge
apt-get build-dep linux
apt-get install git-core libncurses5 libncurses5-dev libelf-dev binutils-dev
apt-get install libibverbs1 libibcm1 libibcm-dev libibverbs-dev ibverbs-utils
apt-get install librdmacm-dev
8、新增配置
vim /etc/udev/rules.d/40-ib.rules
填充如下内容
#### /etc/udev/rules.d/40-ib.rules ####
KERNEL=="umad*",NAME="infiniband/%k"
KERNEL=="issm*",NAME="infiniband/%k"
KERNEL=="ucm*",NAME="infiniband/%k", MODE="0666"
KERNEL=="uverbs*",NAME="infiniband/%k", MODE="0666"
KERNEL=="uat",NAME="infiniband/%k", MODE="0666"
KERNEL=="ucma",NAME="infiniband/%k", MODE="0666"
KERNEL=="rdma_cm",NAME="infiniband/%k", MODE="0666"
########
9、加载内核模块
modprobe rdma_cm
modprobe ib_uverbs
modprobe rdma_ucm
root@ubuntu:~# ls /dev/infiniband/
回显如下:
rdma_cm
10、编译soft-iwarp
apt-get install libtool autoconf
git clone https://github.com/zrlio/softiwarp
# 编译userlib
cd softiwarp/userlib/
./autogen.sh
./configure
make install
ln -s /usr/local/etc/libibverbs.d /etc/libibverbs.d
# 编译kernel模块
cd softiwarp/kernel/
make
mkdir /lib/modules/4.8.17-040817-generic/extra
cp siw.ko /lib/modules/4.8.17-040817-generic/extra
insmod /lib/modules/4.8.17-040817-generic/extra/siw.ko
depmod
modprobe siw
验证:
lsmod | grep rdma
回显如下:
rdma_ucm 28672 0
ib_uverbs 65536 1 rdma_ucm
rdma_cm 57344 2 ib_iser,rdma_ucm
iw_cm 49152 1 rdma_cm
ib_cm 45056 1 rdma_cm
ib_core 212992 7 ib_iser,ib_cm,rdma_cm,ib_uverbs,iw_cm,rdma_ucm,siw
configfs 40960 2 rdma_cm
ls /dev/infiniband/
回显如下:
rdma_cm uverbs0 uverbs1 uverbs2
11、查看IB设备和信息
root@ubuntu:~# ibv_devices
device node GUID
------ ----------------
siw_eth0 52540091e84c0000
siw_lo 7369775f6c6f0000
siw_eth1 52540094d8230000
二、验证
reference: https://github.com/animeshtrivedi/blog/blob/master/post/2019-06-26-siw.md
1、下载rdma-example
git clone https://github.com/animeshtrivedi/rdma-example.git
2、编译
cd rdma-example/src
#新建Makefile
vim Makefile
#填充内容如下
############################
PHONY: clean
CFLAGS := -Wall -Werror -g
LD := gcc
LDLIBS := ${LDLIBS} -lrdmacm -libverbs -lpthread
APPS := rdma_client rdma_server
all: ${APPS}
rdma_client: rdma_common.o rdma_client.o
${LD} -o $@ $^ ${LDLIBS}
rdma_server: rdma_common.o rdma_server.o
${LD} -o $@ $^ ${LDLIBS}
clean:
rm -f *.o ${APPS}
############################
make
3、测试
服务端:启动后服务端阻塞
root@ubuntu:/home/rdma-example/src# ./rdma_server
Server is listening successfully at: 0.0.0.0 , port: 20886
客户端:
root@ubuntu:/home/rdma-example/src# ./rdma_client -a 127.0.0.1 -s teststring
Passed string is : teststring , with count 10
Trying to connect to server at : 127.0.0.1 port: 20886
The client is connected successfully
---------------------------------------------------------
buffer attr, addr: 0x1a021f0 , len: 10 , stag : 0x6ff3785b
---------------------------------------------------------
...
SUCCESS, source and destination buffers match
Client resource clean up is complete
客户端启动后服务端情况如下:
root@ubuntu:/home/rdma-example/src# ./rdma_server
Server is listening successfully at: 0.0.0.0 , port: 20886
A new connection is accepted from 127.0.0.1
Client side buffer information is received...
---------------------------------------------------------
buffer attr, addr: 0x21d8a40 , len: 10 , stag : 0xe6a5f600
---------------------------------------------------------
The client has requested buffer length of : 10 bytes
A disconnect event is received from the client...
Server shut-down is complete
4、其他rdma例程
#这个例子好像在这个模拟环境上有问题,不清楚原因。。
https://thegeekinthecorner.wordpress.com/page/2/