一、系统配置
1.1 主备节点配置host解析
[root@test-128 ~]# cat << EOF >> /etc/hosts
192.168.204.128 test-128
192.168.204.129 test-129
EOF
1.2 DRBD模块安装
备用下载地址:https://download.csdn.net/download/weixin_41558221/89360839
#确认是否加载drbd模块
[root@test-128 ~]# lsmod|grep drbd
drbd 397041 4
libcrc32c 12644 4 xfs,drbd,ip_vs,nf_conntrack
#如未加载
[root@test-128 ~]# rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
[root@test-128 ~]# yum -y install kernel-devel drbd84 kmod-drbd84
#重启加载drbd模块
[root@test-128 ~]# reboot
二、DRBD配置
2.1 drbd配置文件
[root@test-128 ~]# cat << EOF > /etc/drbd.d/rsyncserver.res
resource rsyncserver {
syncer {
rate 1G;
}
disk {
resync-rate 400M;
}
on test-128 {
device /dev/drbd0;
disk /dev/sdb;
address 192.168.204.128:7789;
meta-disk internal;
}
on test-129 {
device /dev/drbd0;
disk /dev/sdb;
address 192.168.204.129:7789;
meta-disk internal;
}
}
EOF
2.2 创建DRBD数据块
#分别在两台机器执行以下操作
[root@test-128 ~]# drbdadm create-md rsyncserver
[root@test-128 ~]# systemctl start drbd
[root@test-128 ~]# systemctl enable drbd
2.3 Master节点配置
[root@test-128 ~]# drbdadm -- --overwrite-data-of-peer primary rsyncserver
[root@test-128 ~]# mkfs.ext4 /dev/drbd0
[root@test-128 ~]# mount /dev/drbd0 /drbd
#查看DRBD状态
[root@test-128 ~]# /lib/drbd/drbd status
2.4 主备连接修复
[root@test-128 ~]# /sbin/drbdadm connect rsyncserver
[root@test-128 ~]# /sbin/drbdadm primary rsyncserver
[root@test-128 ~]# /lib/drbd/drbd status
备节点执行
[root@test-129 ~]# drbdadm secondary rsyncserver
[root@test-129 ~]# drbdadm disconnect rsyncserver
#擦除备节点数据,与主节点同步
[root@test-129 ~]# drbdadm -- --discard-my-data connect rsyncserver
三、Keepalived
3.1 主节点配置
[root@test-128 /tmp]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
script_user root
enable_script_security
}
vrrp_script chk_nfs_port {
script "/etc/keepalived/scripts/check_nfs.sh"
weight -50
rise 3
}
vrrp_instance VI_1 {
state BACKUP
virtual_router_id 66
interface ens33
priority 100
preempt
virtual_ipaddress {
192.168.204.130
}
track_script {
chk_nfs_port
}
notify_master /etc/keepalived/notify_master.sh
notify_backup /etc/keepalived/notify_backup.sh
}
3.2 备节点配置
[root@test-129 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
script_user root
enable_script_security
}
vrrp_script chk_nfs_port {
script "/etc/keepalived/scripts/check_nfs.sh"
weight -50
rise 3
}
vrrp_instance VI_1 {
state BACKUP
virtual_router_id 66
interface eth0
priority 60
preempt
virtual_ipaddress {
192.168.204.130
}
track_script {
chk_nfs_port
}
notify_master /etc/keepalived/notify_master.sh
notify_backup /etc/keepalived/notify_backup.sh
}
3.3 NFS健康检查脚本配置
[root@test-128 /tmp]# cat /etc/keepalived/scripts/check_nfs.sh
#!/bin/bash
CHECK_COUNTS=3
function check_nfs_Runing (){
showmount -e 127.0.0.1 | grep data/drbd &>/dev/null
#showmount -e 127.0.0.1 | grep data/nfs-test &>/dev/null
if [ $? == 0 ] ;then
NFS_OK=0
else
NFS_OK=1
fi
return $NFS_OK
}
while [ $CHECK_COUNTS -ne 0 ]
do
let "CHECK_COUNTS -= 1"
check_nfs_Runing
if [ "$NFS_OK" == "0" ] ; then
exit 0
fi
if [ "$NFS_OK" == "1" ] && [ $CHECK_COUNTS -eq 0 ]
then
exit 1
fi
sleep 1
done
3.4 Backup脚本配置(keepalived切换为backup时执行相应脚本)
[root@test-128 /tmp]# cat /etc/keepalived/notify_backup.sh
#!/bin/bash
function logs(){
if [ $? != 0 ];then
echo -e "\033[31mERROR\033[0m:$1" &>> /etc/keepalived/logs/notify_backup.log
fi
}
time=`date "+%F %H:%M:%S"`
logs "$time ------notify_backup------"
/bin/systemctl stop nfs
logs "Nfs stop"
/bin/systemctl stop rsyncd
logs "Rsyncd stop"
/bin/umount -l /dadsfa
logs "Umount /dev/drbd0"
/sbin/drbdadm secondary rsyncserver
logs "Secondary drbd"
/bin/systemctl start nfs
logs "Nfs stop"
3.5 Master脚本配置(keepalived切换为Master时执行相应脚本)
#!/bin/bash
function logs(){
if [ $? != 0 ];then
echo -e "\033[31mERROR\033[0m:$1" &>> /etc/keepalived/logs/notify_master.log
fi
}
time=`date "+%F %H:%M:%S"`
logs "$time ------notify_backup------"
/sbin/drbdadm primary rsyncserver
logs "Primary drbd"
/bin/mount /dev/drbd0 /data/drbd
logs "Mount /dev/drbd0"
/bin/systemctl restart nfs
logs "Nfs restart"
/bin/systemctl restart rsyncd
logs "Rsyncd restart"
四、监控
4.1 虚拟IP可用
ping
4.2 虚拟IP切换
ip change
4.3 NFS服务可用
showmount -e 127.0.0.1 | grep data/drbd
4.4 Drbd状态
/lib/drbd/drbd status | grep ^0 | awk '{print$2}' == "Connected"
五、异常排查:
备份服务器卸载挂载卷,存在device-busy情况,对应的backup日志为0: State change failed: (-12) Device is held open by someone
查看test-129发现虚拟IP已切换,卸载drbd日志异常。“Device is held open by someone”
对应nfs挂载点均不可用
Primary节点报错
经排查,确定为nfs服务占用。
对应backup脚本卸载/dev/drbd0前,添加"/bin/systemctl stop nfs",以确保正常卸载