通过inotify实现高可用NFS【文件追踪与变更触发同步】

saikey0379

已于 2024-05-25 23:47:36 修改

阅读量349

点赞数 10

文章标签：服务器 linux 运维

于 2024-05-21 23:32:49 首次发布

本文链接：https://blog.csdn.net/weixin_41558221/article/details/139105000

版权

一、NFS配置

1.1 节点【test-128】

[root@test-128 ~]# mkdir /data/nfs-test
[root@test-128 ~]# cat << EOF > /etc/exports
/data/nfs-test 192.168.204.0/24(rw,sync,no_root_squash)
EOF
[root@test-128 ~]# systemctl start nfs;systemctl enable nfs
[root@test-128 ~]# showmount -e 127.0.0.1
Export list for 127.0.0.1:
/data/nfs-test 192.168.204.0/24

1.2 节点【test-129】

[root@test-129 ~]# mkdir /data/nfs-test
[root@test-129 ~]# cat << EOF > /etc/exports
/data/nfs-test 192.168.204.0/24(rw,sync,no_root_squash)
EOF
[root@test-129 ~]# systemctl start nfs;systemctl enable nfs
[root@test-129 ~]# showmount -e 127.0.0.1
Export list for 127.0.0.1:
/data/nfs-test 192.168.204.0/24

二、Rsync配置

2.1 节点【test-128】

[root@test-128 ~]# yum -y install rsync
[root@test-128 ~]# cat <<EOF>/etc/rsyncd.conf 
uid = root
gid = root
use chroot = no
max connections = 200
timeout = 900
 
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /data/logs/rsyncd.log
 
[nfs-test]
path = /data/nfs-test/
ignore errors
read only = false
list = false
hosts allow = 192.168.204.0/24
hosts deny = 0.0.0.0/32
auth users = root
secrets file = /etc/rsyncd.password 
EOF
[root@test-128 ~]# echo "test129" > /etc/rsync.password 
[root@test-128 ~]# echo "root:test128" > /etc/rsyncd.password 
[root@test-128 ~]# chmod 600 /etc/rsync.password
[root@test-128 ~]# chmod 600 /etc/rsyncd.password
[root@test-128 ~]# systemctl start rsyncd && systemctl enable rsyncd

2.2 节点【test-129】

[root@test-129 ~]# yum -y install rsync
[root@test-129 ~]# cat <<EOF>/etc/rsyncd.conf 
uid = root
gid = root
use chroot = no
max connections = 200
timeout = 900
 
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /data/logs/rsyncd.log
 
[nfs-test]
path = /data/nfs-test/
ignore errors
read only = false
list = false
hosts allow = 192.168.204.0/24
hosts deny = 0.0.0.0/32
auth users = root
secrets file = /etc/rsyncd.password 
EOF
[root@test-129 ~]# echo "test128" > /etc/rsync.password 
[root@test-129 ~]# echo "root:test129" > /etc/rsyncd.password 
[root@test-129 ~]# chmod 600 /etc/rsync.password
[root@test-129 ~]# chmod 600 /etc/rsyncd.password
[root@test-129 ~]# systemctl start rsyncd && systemctl enable rsyncd

三、Inotify配置

3.1 安装inotify

[root@test-128 ~]# ls -l /proc/sys/fs/inotify/
-rw-r--r-- 1 root root 0 Sep  2 11:37 max_queued_events
-rw-r--r-- 1 root root 0 Sep  2 11:37 max_user_instances
-rw-r--r-- 1 root root 0 Sep  2 11:37 max_user_watches
[root@test-128 ~]# wget http://github.com/downloads/rvoicilas/inotify-tools/inotify-tools-3.14.tar.gz
[root@test-128 ~]# tar zxvf inotify-tools-3.14.tar.gz && cd inotify-tools-3.14
[root@test-128 ~]# ./configure --prefix=/usr/local/inotify-3.14
[root@test-128 ~]# make && make install
[root@test-128 ~]# cd /usr/local/ && ln -sf inotify-3.14/ inotify

3.2 节点【test-128】

[root@test-128 ~]# cat << "EOF"> /usr/local/inotify-3.14/bin/start.sh 
#!/bin/bash
host01=192.168.204.129
src=/data/nfs-test
dst=nfs-test
user=root
rsync_passfile=/etc/rsync.password
inotify_home=/usr/local/inotify
#judge
if [ ! -e "$src" ] \
|| [ ! -e "${rsync_passfile}" ] \
|| [ ! -e "${inotify_home}/bin/inotifywait" ] \
|| [ ! -e "/usr/bin/rsync" ];
then
echo "Check File and Folder"
exit 9
fi
sleep 5
 
${inotify_home}/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e modify,delete,create,attrib $src \
| while read file
do
cd $src && rsync -auzL --append ./ --timeout=100 $user@$host01::$dst --password-file=${rsync_passfile} >/dev/null 2>&1
done&
exit 0
EOF
[root@test-128 ~]# chmod /usr/local/inotify-3.14/bin/start.sh
[root@test-128 ~]# cat << "EOF"> /lib/systemd/system/inotifyd.service
[Unit]
Description=inotify service
After=network.target
[Service]
Type=forking
Environment=INOHOME=/usr/local/inotify/
ExecStart=/usr/local/inotify/bin/start.sh
ExecStop=
Restart=always
[Install]
WantedBy=multi-user.target
EOF
[root@test-128 ~]# systemctl start inotifyd  && systemctl enable inotifyd

3.3 节点【test-129】

[root@test-129 ~]# cat << "EOF" > /usr/local/inotify-3.14/bin/start.sh 
#!/bin/bash
host01=192.168.204.128
src=/data/nfs-test
dst=nfs-test
user=root 
rsync_passfile=/etc/rsync.password
inotify_home=/usr/local/inotify
#judge
if [ ! -e "$src" ] \
|| [ ! -e "${rsync_passfile}" ] \
|| [ ! -e "${inotify_home}/bin/inotifywait" ] \
|| [ ! -e "/usr/bin/rsync" ];
then
echo "Check File and Folder"
exit 9
fi
sleep 5
 
${inotify_home}/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e modify,delete,create,attrib $src \
| while read file
do
cd $src && rsync -auzL --append ./ --timeout=100 $user@$host01::$dst --password-file=${rsync_passfile} >/dev/null 2>&1
done&
exit 0
EOF
[root@test-129 ~]# chmod /usr/local/inotify-3.14/bin/start.sh
[root@test-129 ~]# cat << "EOF" > /lib/systemd/system/inotifyd.service
[Unit]
Description=inotify service
After=network.target
[Service]
Type=forking
Environment=INOHOME=/usr/local/inotify/
ExecStart=/usr/local/inotify/bin/start.sh
ExecStop=
Restart=always
[Install]
WantedBy=multi-user.target
EOF
[root@test-129 ~]# systemctl start inotifyd  && systemctl enable inotifyd

四、Keepalived配置

4.1 节点【test-128】

[root@test-128 ~]# yum -y install keepalived
[root@test-128 ~]# mkdir /etc/keepalived/scripts/
[root@test-128 ~]# cat << "EOF" >/etc/keepalived/scripts/check_nfs.sh
#!/bin/bash
CHECK_COUNTS=3
  
function check_nfs_Runing (){
showmount -e 127.0.0.1 | grep data/nfs-test &>/dev/null
if [ $? == 0 ] ;then
        NFS_OK=0
else
        NFS_OK=1
fi
return $NFS_OK
  
}
  
while [ $CHECK_COUNTS -ne 0 ]
do
        let "CHECK_COUNTS -= 1"
        check_nfs_Runing
  
        if [ "$NFS_OK" == "0" ] ; then
                exit 0
        fi
  
        if [ "$NFS_OK" == "1" ] &&  [ $CHECK_COUNTS -eq 0 ]
        then
                exit 1
        fi
  
        sleep 1
done
EOF
[root@d3-nfs-001 ~]# chmod +x /etc/keepalived/scripts/check_nfs.sh
[root@test-128 ~]# cat << EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   script_user root
   enable_script_security 
}
vrrp_script chk_nfs_port {
        script "/etc/keepalived/scripts/check_nfs.sh"
        weight -2
        rise 3
}
vrrp_instance VI_1 {
    state BACKUP
    virtual_router_id 66   #ID
    interface ens33 #真实网卡
    priority 100 
    preempt                                                 
    virtual_ipaddress {
        192.168.204.130
    }
    track_script {
      chk_nfs_port                                          
    }
}
EOF
[root@test-128 ~]# systemctl start keepalived  && systemctl enable keepalived

4.2 节点【test-129】

[root@test-129 ~]# yum -y install keepalived
[root@test-129 ~]# mkdir /etc/keepalived/scripts/
[root@test-129 ~]# cat << "EOF" >/etc/keepalived/scripts/check_nfs.sh
#!/bin/bash
CHECK_COUNTS=3
  
function check_nfs_Runing (){
showmount -e 127.0.0.1 | grep data/nfs-test &>/dev/null
if [ $? == 0 ] ;then
        NFS_OK=0
else
        NFS_OK=1
fi
return $NFS_OK
  
}
  
while [ $CHECK_COUNTS -ne 0 ]
do
        let "CHECK_COUNTS -= 1"
        check_nfs_Runing
  
        if [ "$NFS_OK" == "0" ] ; then
                exit 0
        fi
  
        if [ "$NFS_OK" == "1" ] &&  [ $CHECK_COUNTS -eq 0 ]
        then
                exit 1
        fi
  
        sleep 1
done
EOF
[root@test-129 ~]# chmod +x /etc/keepalived/scripts/check_nfs.sh
[root@test-129 ~]# cat << EOF > /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   script_user root
   enable_script_security 
}

vrrp_script chk_nfs_port {
        script "/etc/keepalived/scripts/check_nfs.sh"
        weight -2
        rise 3
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 66
    priority 99
    virtual_ipaddress {
        192.168.204.130
    }
    track_script {
      chk_nfs_port                                          
    }
}
EOF
[root@test-129 ~]# systemctl start keepalived  && systemctl enable keepalived

五、数据维护(清理)

注：rsync采用增量同步，故需手动进行数据清理。

5.1 停止inotifyd

[root@test-128 ~]# systemctl stop inotifyd
#129同上

5.2 执行操作

[root@test-128 ~]# rm -f /data/nfs-test/test
#129同上

5.3 启动inotifyd

[root@test-128 ~]# systemctl start inotifyd
#129同上

附：

#!/bin/bash
host01=192.168.0.1
host02=192.168.0.2
src=/data/www/
dst=data-www
inotify_home=/usr/local/inotify
#judge
if [ ! -e "$src" ] \
|| [ ! -e "${inotify_home}/bin/inotifywait" ] \
|| [ ! -e "/usr/bin/rsync" ];
then
echo "Check File and Folder"
exit 9
fi
sleep 5

${inotify_home}/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e modify,delete,create,attrib $src \
| while read file
do
cd $src
process01=$(ps -ef | grep rsync| grep -v grep| grep -E "$host01::$dst")
echo ${process01}
if [ -z "${process01}" ];then
    rsync                  -aruzL --delete ./ --timeout=100 --exclude=default $host01::$dst --password-file=${rsync_passfile} >/dev/null 2>&1 &
fi

echo ${process02}
process02=$(ps -ef | grep rsync| grep -v grep| grep -E "$host02::$dst")
if [ -z "${process02}" ];then
ps -ef | grep rsync| grep -v grep| grep -E "$host02::$dst" || rsync -e 'ssh -p 5837' -aruzL --delete ./ --timeout=100 --exclude=default $host02::$dst --password-file=${rsync_passfile} >/dev/null 2>&1 &
fi
done&
exit 0

cat /usr/lib/systemd/system/inotifyd_www.service
[Unit]
Description=inotify service
After=network.target
[Service]
Type=forking
Environment=INOHOME=/usr/local/inotify/
ExecStart=/usr/local/inotify/bin/inotify_www.sh
ExecStop=
Restart=always
[Install]
WantedBy=multi-user.target