通过DRBD创建高可用NFS服务

一、系统配置

1.1 主备节点配置host解析

[root@test-128 ~]# cat << EOF >> /etc/hosts
192.168.204.128 test-128
192.168.204.129 test-129
EOF

1.2 DRBD模块安装

备用下载地址：https://download.csdn.net/download/weixin_41558221/89360839

#确认是否加载drbd模块
[root@test-128 ~]# lsmod|grep drbd   
drbd                  397041  4 
libcrc32c              12644  4 xfs,drbd,ip_vs,nf_conntrack
#如未加载
[root@test-128 ~]# rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
[root@test-128 ~]# yum -y install kernel-devel drbd84 kmod-drbd84
#重启加载drbd模块
[root@test-128 ~]# reboot

二、DRBD配置

2.1 drbd配置文件

[root@test-128 ~]# cat << EOF > /etc/drbd.d/rsyncserver.res  
resource rsyncserver {
    syncer {
        rate 1G;
    }
    disk {
        resync-rate 400M;
    }
    on test-128 {
        device /dev/drbd0;
        disk /dev/sdb;
        address 192.168.204.128:7789;
        meta-disk internal;
    }
    on test-129 {
        device /dev/drbd0;
        disk /dev/sdb;
        address 192.168.204.129:7789;
        meta-disk internal;
    }
}
EOF

2.2 创建DRBD数据块

#分别在两台机器执行以下操作
[root@test-128 ~]# drbdadm create-md rsyncserver 
[root@test-128 ~]# systemctl start drbd
[root@test-128 ~]# systemctl enable drbd

2.3 Master节点配置

[root@test-128 ~]# drbdadm -- --overwrite-data-of-peer primary  rsyncserver
[root@test-128 ~]# mkfs.ext4 /dev/drbd0
[root@test-128 ~]# mount /dev/drbd0 /drbd
#查看DRBD状态
[root@test-128 ~]# /lib/drbd/drbd status

2.4 主备连接修复

[root@test-128 ~]# /sbin/drbdadm connect rsyncserver
[root@test-128 ~]# /sbin/drbdadm primary rsyncserver
[root@test-128 ~]# /lib/drbd/drbd status 
备节点执行
[root@test-129 ~]# drbdadm secondary rsyncserver 
[root@test-129 ~]# drbdadm disconnect rsyncserver
#擦除备节点数据，与主节点同步
[root@test-129 ~]# drbdadm -- --discard-my-data connect rsyncserver

三、Keepalived

3.1 主节点配置

[root@test-128 /tmp]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
   script_user root
   enable_script_security 
}
vrrp_script chk_nfs_port {
        script "/etc/keepalived/scripts/check_nfs.sh"
        weight -50
        rise 3
}
vrrp_instance VI_1 {
    state BACKUP
    virtual_router_id 66
    interface ens33
    priority 100
    preempt                                                 
    virtual_ipaddress {
        192.168.204.130
    }
    track_script {
      chk_nfs_port                                          
    }
    notify_master /etc/keepalived/notify_master.sh
    notify_backup /etc/keepalived/notify_backup.sh
}

3.2 备节点配置

[root@test-129 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
   script_user root
   enable_script_security 
}
vrrp_script chk_nfs_port {
        script "/etc/keepalived/scripts/check_nfs.sh"
        weight -50
        rise 3
}
vrrp_instance VI_1 {
    state BACKUP
    virtual_router_id 66
    interface eth0
    priority 60
    preempt
    virtual_ipaddress {
        192.168.204.130
    }
    track_script {
      chk_nfs_port                                          
    }
    notify_master /etc/keepalived/notify_master.sh
    notify_backup /etc/keepalived/notify_backup.sh
}

3.3 NFS健康检查脚本配置

[root@test-128 /tmp]# cat /etc/keepalived/scripts/check_nfs.sh
#!/bin/bash
CHECK_COUNTS=3
   
function check_nfs_Runing (){
showmount -e 127.0.0.1 | grep data/drbd &>/dev/null
#showmount -e 127.0.0.1 | grep data/nfs-test &>/dev/null
if [ $? == 0 ] ;then
        NFS_OK=0
else
        NFS_OK=1
fi
return $NFS_OK
   
}
   
while [ $CHECK_COUNTS -ne 0 ]
do
        let "CHECK_COUNTS -= 1"
        check_nfs_Runing
   
        if [ "$NFS_OK" == "0" ] ; then
                exit 0
        fi
   
        if [ "$NFS_OK" == "1" ] &&  [ $CHECK_COUNTS -eq 0 ]
        then
                exit 1
        fi
   
        sleep 1
done

3.4 Backup脚本配置（keepalived切换为backup时执行相应脚本）

[root@test-128 /tmp]# cat /etc/keepalived/notify_backup.sh      
#!/bin/bash

function logs(){
if [ $? != 0 ];then
  echo -e "\033[31mERROR\033[0m:$1" &>> /etc/keepalived/logs/notify_backup.log
fi
}

time=`date "+%F  %H:%M:%S"`
logs "$time  ------notify_backup------"

/bin/systemctl stop  nfs            
logs "Nfs stop"

/bin/systemctl stop  rsyncd
logs "Rsyncd stop" 

/bin/umount -l /dadsfa
logs "Umount /dev/drbd0"

/sbin/drbdadm secondary rsyncserver
logs "Secondary drbd"

/bin/systemctl start nfs 
logs "Nfs stop"

3.5 Master脚本配置（keepalived切换为Master时执行相应脚本）

#!/bin/bash

function logs(){
if [ $? != 0 ];then
  echo -e "\033[31mERROR\033[0m:$1" &>> /etc/keepalived/logs/notify_master.log
fi
}

time=`date "+%F  %H:%M:%S"`
logs "$time  ------notify_backup------"

/sbin/drbdadm primary rsyncserver
logs "Primary drbd"

/bin/mount /dev/drbd0 /data/drbd
logs "Mount /dev/drbd0"

/bin/systemctl restart  nfs
logs "Nfs restart"

/bin/systemctl restart  rsyncd 
logs "Rsyncd restart"

四、监控

4.1 虚拟IP可用

ping

4.2 虚拟IP切换

ip change

4.3 NFS服务可用

showmount -e 127.0.0.1 | grep data/drbd

4.4 Drbd状态

/lib/drbd/drbd status | grep ^0 | awk '{print$2}' == "Connected"

五、异常排查：

备份服务器卸载挂载卷，存在device-busy情况，对应的backup日志为0: State change failed: (-12) Device is held open by someone

查看test-129发现虚拟IP已切换，卸载drbd日志异常。“Device is held open by someone”

对应nfs挂载点均不可用

Primary节点报错

经排查，确定为nfs服务占用。

对应backup脚本卸载/dev/drbd0前，添加"/bin/systemctl stop nfs",以确保正常卸载