Centos7下利用convoy+nfs实现container之间的共享存储

操作环境

[root@swarm-node1]# cat /etc/redhat-release 
CentOS Linux release 7.4.1708 (Core) 

[root@swarm-node1 ~]# docker version
Client:
 Version:      1.13.1
 API version:  1.26
 Go version:   go1.7.5
 Git commit:   092cba3
 Built:        Wed Feb  8 06:38:28 2017
 OS/Arch:      linux/amd64

Server:
 Version:      1.13.1
 API version:  1.26 (minimum version 1.12)
 Go version:   go1.7.5
 Git commit:   092cba3
 Built:        Wed Feb  8 06:38:28 2017
 OS/Arch:      linux/amd64
 Experimental: false

网络拓扑图

两个Docker node分别连接NFS Server，并在node上安装convoy插件。

配置步骤

配置NFS Server

参考 配置Docker使用Convoy存储插件中配置nfs server的步骤

配置Convoy插件

1.这里配置nfs的存储路径为10.10.200.224:/docker-nfs，分别在docker node1以及node2上创建docker-nfs目录，用来挂载nfs server

#mount -t nfs 10.10.200.224:/docker-nfs /docker-nfs

在fstab中配置nfs自动挂载

#
# /etc/fstab
# Created by anaconda on Thu Dec  7 01:30:28 2017
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/centos-root /                       xfs     defaults        1 1
UUID=3a754ff1-9fd8-4879-a1fb-07fbeebf0da4 /boot                   xfs     defaults        1 2
/dev/mapper/centos-swap swap                    swap    defaults        0 0
10.10.200.224:/docker-nfs /docker-nfs           nfs     defaults        1 1

2.安装convoy

参考  配置Docker使用Convoy存储插件中配置convoy的步骤

3.分别在node1,、node2上启动convoy

#convoy daemon --drivers vfs --driver-opts vfs.path=/docker-nfs 

配置convoy daemon开机启动

编写convoy.sh脚本

[root@swarm-node1 ~]# vi convoy.sh 
#!/bin/bash
convoy daemon --drivers vfs --driver-opts vfs.path=/docker-nfs

分别对/etc/rc.d/rc.local以及convoy.sh添加权限

[root@swarm-node1 ~]# chmod a+x convoy.sh 
[root@swarm-node1 ~]# chmod +x /etc/rc.d/rc.local 

在/etc/rc.d/rc.local中添加convoy.sh启动脚本

#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.

touch /var/lock/subsys/local
/root/convoy.sh

4.我们在node1上启动container：centos-2，使用convoy存储插件，卷名为datavol2

[root@swarm-node1 docker-nfs]# docker run -ti --volume-driver=convoy -v datavol2:/data --name centos-2 --hostname centos-2 centos
[root@centos-2 /]# df
Filesystem                         1K-blocks    Used Available Use% Mounted on
overlay                            957611164 3392348 954218816   1% /
tmpfs                                3996928       0   3996928   0% /dev
tmpfs                                3996928       0   3996928   0% /sys/fs/cgroup
10.10.200.224:/docker-nfs/datavol2  49775104 4192768  45582336   9% /data
/dev/mapper/centos-root            957611164 3392348 954218816   1% /etc/hosts
shm                                    65536       0     65536   0% /dev/shm
tmpfs                                3996928       0   3996928   0% /sys/firmware

我们在node1上，查看convoy 卷

[root@swarm-node1 docker-nfs]# convoy list
{
......
        "datavol1": {
                "Name": "datavol1",
                "Driver": "vfs",
                "MountPoint": "/docker-nfs/datavol1",
                "CreatedTime": "Wed Dec 20 14:16:41 +0800 2017",
                "DriverInfo": {
                        "Driver": "vfs",
                        "MountPoint": "/docker-nfs/datavol1",
                        "Path": "/docker-nfs/datavol1",
                        "PrepareForVM": "false",
                        "Size": "0",
                        "VolumeCreatedAt": "Wed Dec 20 14:16:41 +0800 2017",
                        "VolumeName": "datavol1"
                },
                "Snapshots": {}
        },
        "datavol2": {
                "Name": "datavol2",
                "Driver": "vfs",
                "MountPoint": "/docker-nfs/datavol2",
                "CreatedTime": "Wed Dec 20 14:58:40 +0800 2017",
                "DriverInfo": {
                        "Driver": "vfs",
                        "MountPoint": "/docker-nfs/datavol2",
                        "Path": "/docker-nfs/datavol2",
                        "PrepareForVM": "false",
                        "Size": "0",
                        "VolumeCreatedAt": "Wed Dec 20 14:58:40 +0800 2017",
                        "VolumeName": "datavol2"
                },
                "Snapshots": {}
......
}

再node2上查看convoy，这里我们使用docker volume来查询，同样可以查询到datavol2

[root@swarm-node2 docker-nfs]# docker volume ls -f driver=convoy
DRIVER              VOLUME NAME
......
convoy              datavol1
convoy              datavol2
......

在node2上，使用datavol2启动container

[root@swarm-node2 docker-nfs]# docker run -ti --volume-driver=convoy -v datavol2:/data --name centos-2 --hostname centos-2 centos
[root@centos-2 /]# df
Filesystem                                                                                     1K-blocks    Used Available Use% Mounted on
/dev/mapper/docker-253:1-3488-c354148820fa89ab5f5d8bb1ece67d4607da28164cd4eac02718c7ffcdefedf0  10473472  253648  10219824   3% /
tmpfs                                                                                            8125696       0   8125696   0% /dev
tmpfs                                                                                            8125696       0   8125696   0% /sys/fs/cgroup
10.10.200.224:/docker-nfs/datavol2                                                              49775104 4193024  45582080   9% /data
/dev/mapper/centos-root                                                                        948886852 2104060 946782792   1% /etc/hosts
shm                                                                                                65536       0     65536   0% /dev/shm
tmpfs                                                                                            8125696       0   8125696   0% /sys/firmware

现在分别在node1、node2上使用同一个volume启动了2个container，我们在node1上的container中写入数据

[root@swarm-node1 docker-nfs]# docker attach centos-2
[root@centos-2 /]# touch /data/test
[root@centos-2 /]# echo hello_world > /data/test 
[root@centos-2 /]# cat /data/test    
hello_world

在进入node2中的container，看数据是否存在

[root@swarm-node2 docker-nfs]# docker attach centos-2
[root@centos-2 /]# cat /data/test 
hello_world

数据存在，这样就完成了，通过convoy+nfs实现了container之间的共享存储