Anolis 8 NVME over TCP 配置使用

NVMe over TCP，简单来说就是使用TCP协议下发NVMe命令给远端的NVMe设备。

NVMe over TCP环境搭建时，发送NVMe命令的叫做host端，接收NVMe命令的叫target端。host 端挂载target端的磁盘到本地使用。

基于 Anolis OS release 8.6 kernel 5.10.134-13.an8.x86_64

1. 检查内核配置

在开始之前需要确定NVME_TCP、NVME_TARGET等内核配置是被set的, host和target端都得检查并支持

# cat /boot/config-`uname -r` | grep NVME

# NVME Support
CONFIG_NVME_CORE=m
CONFIG_BLK_DEV_NVME=m
CONFIG_NVME_MULTIPATH=y
# CONFIG_NVME_HWMON is not set
CONFIG_NVME_FABRICS=m
CONFIG_NVME_RDMA=m
CONFIG_NVME_FC=m
CONFIG_NVME_TCP=m
CONFIG_NVME_TARGET=m
# CONFIG_NVME_TARGET_PASSTHRU is not set
CONFIG_NVME_TARGET_LOOP=m
CONFIG_NVME_TARGET_RDMA=m
CONFIG_NVME_TARGET_FC=m
CONFIG_NVME_TARGET_FCLOOP=m
CONFIG_NVME_TARGET_TCP=m
# end of NVME Support
CONFIG_RTC_NVMEM=y
CONFIG_NVMEM=y
CONFIG_NVMEM_SYSFS=y

2. 安装nvme-cli

host和target端安装nvme-cli

#yum -y install nvme-cli

3. target 端配置

target端查看nqn和hostid

# nvme list
Node             SN                   Model                                    Namespace Usage                      Format           FW Rev  
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1     P70316002175         P45C7016D22                              1           1.60  TB /   1.60  TB    512   B +  0 B   0090500
# cat /etc/nvme/hostnqn   
nqn.2014-08.org.nvmexpress:uuid:ffffffff-ffff-ffff-ffff-ffffffffffff
# cat /etc/nvme/hostid 
582b6e98-59cc-484e-a8f0-d997350b2be4

# modprobe nvmet
# modprobe nvmet-tcp
# ls /sys/kernel/config/nvmet/
hosts  ports  subsystems
#cd /sys/kernel/config/nvmet/subsystems
#mkdir nvme-test-target
#tree
.
└── nvme-test-target
    ├── allowed_hosts
    ├── attr_allow_any_host
    ├── attr_serial
    ├── attr_version
    └── namespaces

3 directories, 3 files

#cd nvme-test-target/
#允许所有主机访问
#echo 1 > attr_allow_any_host 

#nvme list
Node             SN                   Model                                    Namespace Usage                      Format           FW Rev  
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1     P70316002175         P45C7016D22                              1           1.60  TB /   1.60  TB    512   B +  0 B   0090500

#可以看出/dev/nvme0n1 是namespace 1
#mkdir namespaces/1
#ll namespaces/1
total 0
-rw-r--r-- 1 root root 4096 Sep  3 20:17 device_nguid
-rw-r--r-- 1 root root 4096 Sep  3 20:17 device_path
-rw-r--r-- 1 root root 4096 Sep  3 20:17 device_uuid
-rw-r--r-- 1 root root 4096 Sep  3 20:17 enable

指定namespace 1的device并启用
#echo '/dev/nvme0n1' > namespaces/1/device_path
#echo 1 > namespaces/1/enable

创建端口号，配置IP和其他参数

#mkdir /sys/kernel/config/nvmet/ports/1
#cd /sys/kernel/config/nvmet/ports/1
#ll
total 0
-rw-r--r-- 1 root root 4096 Sep  3 20:21 addr_adrfam
-rw-r--r-- 1 root root 4096 Sep  3 20:21 addr_traddr
-rw-r--r-- 1 root root 4096 Sep  3 20:21 addr_treq
-rw-r--r-- 1 root root 4096 Sep  3 20:21 addr_trsvcid
-rw-r--r-- 1 root root 4096 Sep  3 20:21 addr_trtype
-rw-r--r-- 1 root root 4096 Sep  3 20:21 param_inline_data_size
drwxr-xr-x 2 root root    0 Sep  3 20:21 referrals
drwxr-xr-x 2 root root    0 Sep  3 20:21 subsystems
#echo '10.203.173.30' > addr_traddr 

#低版本内核 4.19* 下面报过错误，升级5.*后正常
#echo tcp > addr_trtype
#echo 4420 > addr_trsvcid 
#echo ipv4 > addr_adrfam

将端口与子系统绑定

#ln -s /sys/kernel/config/nvmet/subsystems/nvme-test-target/ /sys/kernel/config/nvmet/ports/1/subsystems/nvme-test-target

#dmesg -T|grep -i nvme
[Thu Sep  5 15:24:20 2024] nvme nvme0: pci function 0000:84:00.0
[Thu Sep  5 15:24:23 2024] nvme nvme0: 24/0/0 default/read/poll queues
[Thu Sep  5 15:38:36 2024] nvmet: adding nsid 1 to subsystem nvme-test-target
[Thu Sep  5 15:41:01 2024] nvmet_tcp: enabling port 1 (10.203.173.30:4420)

#可以看到nvme_tcp服务被启动
#netstat -ntlp
tcp        0      0 10.203.173.30:4420      0.0.0.0:*               LISTEN      -                

4. host端配置

安装nvme、nvmet-tcp模块

# modprobe nvmet
# modprobe nvmet-tcp

discovery搜索target节点磁盘

#nvme discover -t tcp -a 10.203.173.30 -s 4420
Discovery Log Number of Records 1, Generation counter 2
=====Discovery Log Entry 0======
trtype:  tcp
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified, sq flow control disable supported
portid:  1
trsvcid: 4420
subnqn:  nvme-test-target
traddr:  10.203.173.30
sectype: none

host挂载target磁盘

挂载前
#nvme list
Node                  SN                   Model                                    Namespace Usage                      Format           FW Rev  
--------------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------

挂载
#nvme connect -t tcp -n nvme-test-target -a 10.203.173.30 -s 4420 --hostnqn=nqn.2014-08.org.nvmexpress:uuid:ffffffff-ffff-ffff-ffff-ffffffffffff

挂载后
#nvme list                                                                                                                                      
Node                  SN                   Model                                    Namespace Usage                      Format           FW Rev  
--------------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1          2bbbe276ca0dbd62     Linux                                    1           1.60  TB /   1.60  TB    512   B +  0 B   5.10.134

此时host 已经挂载上了target的nvme磁盘，target端也有了对应的连接

# netstat -antlp|grep 4420
tcp        0      0 10.203.173.30:4420      0.0.0.0:*               LISTEN      -                   
tcp        0      0 10.203.173.30:4420      10.53.178.139:58140     ESTABLISHED -                   
tcp        0      0 10.203.173.30:4420      10.53.178.139:58134     ESTABLISHED -                   
tcp        0      0 10.203.173.30:4420      10.53.178.139:58150     ESTABLISHED -   

断开连接

#nvme disconnect -n nvme-test-target  

卸载target

移除软连接

#rm -rf /sys/kernel/config/nvmet/ports/1/subsystems/nvme-test-target

关闭target解绑设备

#echo 0 > /sys/kernel/config/nvmet/subsystems/nvme-test-target/namespaces/1/enable
#echo -n 0 > /sys/kernel/config/nvmet/subsystems/nvme-test-target/namespaces/1/device_path

移除transport层

#rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/ports/1

移除NVME subsystem

#rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/subsystems/nvme-test-target/namespaces/1
#rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/subsystems/nvme-test-target