keepalived+DRBD+NFS高可用共享存储
一、BRBD 环境部署
环境准备
配置 hosts解析
---node01
[root@node02 ~]# vim /etc/hosts +
192.168.5.11 node01
192.168.5.12 node02
---node02
[root@node02 ~]# vim /etc/hosts +
192.168.5.11 node01
192.168.5.12 node02
配置 免密钥互信
---node01
[root@node01 ~]# ssh-keygen -f ~/.ssh/id_rsa -P '' -q
[root@node01 ~]# ssh-copy-id node01
[root@node01 ~]# ssh-copy-id node02
---node02
[root@node02 ~]# ssh-keygen -f ~/.ssh/id_rsa -P '' -q
[root@node02 ~]# ssh-copy-id node01
[root@node02 ~]# ssh-copy-id node02
配置 DRBD yum 源
---node01
[root@node01 ~]# vim /etc/yum.repos.d/drbd.repo
[drbd]
name=drbd
baseurl=https://mirrors.tuna.tsinghua.edu.cn/elrepo/elrepo/el7/x86_64/
gpgcheck=0
---node02
[root@node02 ~]# vim /etc/yum.repos.d/drbd.repo
[drbd]
name=drbd
baseurl=https://mirrors.tuna.tsinghua.edu.cn/elrepo/elrepo/el7/x86_64/
gpgcheck=0
安装 DRBD
[root@node01 ~]# yum install -y drbd84-utils kmod-drbd84
[root@node01 ~]# yum install -y drbd84-utils kmod-drbd84
加载内核
---node01
[root@node01 ~]# lsmod | grep drbd
[root@node01 ~]# modprobe drbd
[root@node01 ~]# lsmod | grep drbd
drbd 397041 0
libcrc32c 12644 2 xfs,drbd
---node02
[root@node02 ~]# lsmod | grep drbd
[root@node02 ~]# modprobe drbd
[root@node02 ~]# lsmod | grep drbd
drbd 397041 0
libcrc32c 12644 2 xfs,drbd
修改 DRBD 配置文件
[root@node01 ~]# vim /etc/drbd.d/global_common.conf
# DRBD is the result of over a decade of development by LINBIT.
# In case you need professional services for DRBD or have
# feature requests visit http://www.linbit.com
global {
usage-count no;
# Decide what kind of udev symlinks you want for "implicit" volumes
# (those without explicit volume <vnr> {} block, implied vnr=0):
# /dev/drbd/by-resource/<resource>/<vnr> (explicit volumes)
# /dev/drbd/by-resource/<resource> (default for implict)
udev-always-use-vnr; # treat implicit the same as explicit volumes
# minor-count dialog-refresh disable-ip-verification
# cmd-timeout-short 5; cmd-timeout-medium 121; cmd-timeout-long 600;
}
common {
protocol C;
handlers {
# These are EXAMPLE handlers only.
# They may have severe implications,
# like hard resetting the node under certain circumstances.
# Be careful when choosing your poison.
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
# fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
# split-brain "/usr/lib/drbd/notify-split-brain.sh root";
# out-of-sync "/usr/lib/drbd/notify-out-of-sync.sh root";
# before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
# after-resync-target /usr/lib/drbd/unsnapshot-resync-target-lvm.sh;
# quorum-lost "/usr/lib/drbd/notify-quorum-lost.sh root";
}
startup {
# wfc-timeout degr-wfc-timeout outdated-wfc-timeout wait-after-sb
}
options {
# cpu-mask on-no-data-accessible
# RECOMMENDED for three or more storage nodes with DRBD 9:
# quorum majority;
# on-no-quorum suspend-io | io-error;
}
disk {
on-io-error detach;
# size on-io-error fencing disk-barrier disk-flushes
# disk-drain md-flushes resync-rate resync-after al-extents
# c-plan-ahead c-delay-target c-fill-target c-max-rate
# c-min-rate disk-timeout
}
net {
cram-hmac-alg "sha1";
shared-secret "nfs-HA";
allow-two-primaries;
}
syncer { rate 1000M; }
}
[root@node01 ~]# vim /etc/drbd.d/nfs.res
resource nfs {
meta-disk internal;
device /dev/drbd1;
disk /dev/nfs/share;
on node01 {
address 192.168.5.11:7789;
}
on node02 {
address 192.168.5.12:7789;
}
}
[root@node01 ~]# scp /etc/drbd.d/* node02:/etc/drbd.d/
LVM 格式化
---node01
[root@node01 ~]# vgcreate nfs /dev/sdb
[root@node01 ~]# lvcreate --size 2G --name share nfs
---node02
[root@node01 ~]# vgcreate nfs /dev/sdb
[root@node01 ~]# lvcreate --size 2G --name share nfs
创建用户
---node01
[root@node01 ~]# useradd -M -s /sbin/nologin haclient
[root@node01 ~]# chgrp haclient /lib/drbd/drbdsetup-84
[root@node01 ~]# chmod o-x /lib/drbd/drbdsetup-84
[root@node01 ~]# chmod u+s /lib/drbd/drbdsetup-84
[root@node01 ~]# chgrp haclient /usr/sbin/drbdmeta
[root@node01 ~]# chmod o-x /usr/sbin/drbdmeta
[root@node01 ~]# chmod u+s /usr/sbin/drbdmeta
---node02
[root@node02 ~]# useradd -M -s /sbin/nologin haclient
[root@node02 ~]# chgrp haclient /lib/drbd/drbdsetup-84
[root@node02 ~]# chmod o-x /lib/drbd/drbdsetup-84
[root@node02 ~]# chmod u+s /lib/drbd/drbdsetup-84
[root@node02 ~]# chgrp haclient /usr/sbin/drbdmeta
[root@node02 ~]# chmod o-x /usr/sbin/drbdmeta
[root@node02 ~]# chmod u+s /usr/sbin/drbdmeta
激活资源
[root@node01 ~]# drbdadm create-md data
[root@node02 ~]# drbdadm create-md data
查看 DRBD 状态
node01
node02
node01 强制升级
drbdadm -- --force primary nfs
再次查看 DRBD 状态
node01
node02
格式化、挂载 DRBD
[root@node01 ~]# mkdir data
[root@node01 ~]# mkfs.xfs /dev/drbd1
[root@node01 ~]# mount /dev/drbd1 data
查看挂载
DRBD 挂载 网络数据同步情况
[root@node01 ~]# echo "123" > data/a.txt
[root@node01 ~]# cat data/a.txt
123
二、配置 NFS 共享存储
---node01
[root@node01 ~]# yum -y install rpcbind nfs-utils
[root@node01 ~]# vim /etc/exports
[root@node01 ~]# systemctl start rpcbind
[root@node01 ~]# systemctl start nfs
---node02
[root@node02 ~]# yum -y install rpcbind nfs-utils
[root@node02 ~]# systemctl start rpcbind
[root@node02 ~]# systemctl start nfs
三、Keepalived 实现集群高可用
安装 keepalived
[root@node01 ~]# yum install keepalived -y
[root@node02 ~]# yum install keepalived -y
修改keepalived 配置文件
[root@node01 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DRBD_HA_MASTER
}
vrrp_script chk_nfs {
script "/etc/keepalived/check_nfs.sh"
interval 5
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
track_script {
chk_nfs
}
notify_stop /etc/keepalived/notify_stop.sh
notify_master /etc/keepalived/notify_master.sh
virtual_ipaddress {
192.168.5.100
}
}
提供 Keepalived + DRBD 集群脚本
---node01
[root@node01 ~]# mkdir /etc/keepalived/logs
[root@node01 ~]# vim /etc/keepalived/check_nfs.sh
/sbin/service nfs status &>/dev/null
if [ $? -ne 0 ];then
/sbin/service nfs restart
/sbin/service nfs status &>/dev/null
if [ $? -ne 0 ];then
umount /dev/drbd1
drbdadm secondary nfs
/sbin/service keepalived stop
fi
fi
[root@node01 ~]# vim /etc/keepalived/notify_master.sh
#!/bin/bash
time=`date "+%F %T"`
echo -e "$time ------notify_master------\n" >> /etc/keepalived/logs/notify_master.log
/sbin/drbdadm primary nfs &>> /etc/keepalived/logs/notify_master.log
/bin/mount /dev/drbd1 data &>> /etc/keepalived/logs/notify_master.log
/sbin/service nfs restart &>> /etc/keepalived/logs/notify_master.log
echo -e "\n" >> /etc/keepalived/logs/notify_master.log
[root@node01 ~]# vim /etc/keepalived/notify_stop.sh
#!/bin/bash
time=`date "+%F %H:%M:%S"`
echo -e "$time ------notify_stop------\n" >> /etc/keepalived/logs/notify_stop.log
/sbin/service nfs stop &>> /etc/keepalived/logs/notify_stop.log
/bin/umount data &>> /etc/keepalived/logs/notify_stop.log
/sbin/drbdadm secondary nfs &>> /etc/keepalived/logs/notify_stop.log
echo -e "\n" >> /etc/keepalived/logs/notify_stop.log
[root@node01 ~]# chmod +x /etc/keepalived/*.sh
[root@node01 ~]# ll /etc/keepalived/*.sh
-rwxr-xr-x 1 root root 248 Mar 9 14:06 /etc/keepalived/check_nfs.sh
-rwxr-xr-x 1 root root 389 Mar 9 14:07 /etc/keepalived/notify_master.sh
-rwxr-xr-x 1 root root 367 Mar 9 14:09 /etc/keepalived/notify_stop.sh
[root@node01 ~]# scp /etc/keepalived/* node02:/etc/keepalived/
---node02
[root@node02 ~]# mkdir /etc/keepalived/logs
启动 keepalived 服务
[root@node01 ~]# systemctl start keepalived.service
[root@node02 ~]# systemctl start keepalived.service
查看 VIP
node01
node02
四、测试
1、模拟 NFS 服务故障
node01
node02
实现 NFS 故障 恢复!
2、测试 keepalived 故障
MASTER keepalived 故障
node02
MASTER keepalived 故障恢复
node02
实现 Keepalived VIP 漂移!