OCFS2+NFS+Keepalived安装配置-oracle linux上的高可用存储

1 环境准备

两台Oracle linux 7 设备

1.1 ip分配

host	ip
host-nfs-1722012840	172.20.128.40
host-nfs-1722012841	172.20.128.41
vip	172.20.128.42

1.2 oracle配置本地yum源

• 公司是内网，配置了本地yum源，可以在阿里云镜像站搜索阿里云yum源配置方法

mount /dev/sr0 /media  

cd /etc/yum.repos.d/
mkdir bak
mv *.repo bak/
vi oracle-linux-ol7.repo
[ol7_latest]
name=Oracle Linux $releasever Latest ($basearch)
baseurl=file:///media
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
gpgcheck=1
enabled=1

yum clean all
yum makecache

# 下tomcat测试下
yum -y install tomcat
yum -y remove t

yum install -y kernel-uek ocfs2* psmisc*

1.3 hostnamectl

hostnamectl set-hostname host-nfs-1722012840
hostnamectl set-hostname host-nfs-1722012841

cat>>/etc/hosts<<EOF
172.20.128.40 host-nfs-1722012840
172.20.128.41 host-nfs-1722012841
EOF

1.4 firewalld、selinux

systemctl stop firewalld
systemctl disable firewalld
sed -i '/SELINUX/s/enforcing/disabled/g' /etc/selinux/config
setenforce 0
getenforce

2 oscf

2.1 加入节点

mkdir -p /dlm

# 第一节点执行
o2cb add-cluster ocfsstorage
o2cb add-node ocfsstorage host-nfs-1722012840 --ip 172.20.128.40
o2cb add-node ocfsstorage host-nfs-1722012841 --ip 172.20.128.41
o2cb heartbeat-mode ocfsstorage local

cat /etc/ocfs2/cluster.conf

2.2 o2cb.init configure

# 两个节点执行
o2cb.init configure
o2cb.init status

o2cb.init configure
# o2cb.init configure
Configuring the O2CB driver.
This will configure the on-boot properties of the O2CB driver.
The following questions will determine whether the driver is loaded on
boot. The current values will be shown in brackets ('[]'). Hitting
<ENTER> without typing an answer will keep that current value. Ctrl-C
will abort.
Load O2CB driver on boot (y/n) [n]: y
Cluster stack backing O2CB [o2cb]:
Cluster to start on boot (Enter "none" to clear) [ocfs2]: ocfsstorage
Specify heartbeat dead threshold (>=7) [31]: 121
Specify network idle timeout in ms (>=5000) [30000]:
Specify network keepalive delay in ms (>=1000) [2000]: 5000
Specify network reconnect delay in ms (>=2000) [2000]: 4000
Writing O2CB configuration: OK
checking debugfs...
Loading stack plugin "o2cb": OK
Loading filesystem "ocfs2_dlmfs": OK
Mounting ocfs2_dlmfs filesystem at /dlm: OK
Setting cluster stack "o2cb": OK
Registering O2CB cluster "ocfsstorage": OK
Setting O2CB cluster timeouts : OK
# o2cb.init status
Driver for "configfs": Loaded
Filesystem "configfs": Mounted
Stack glue driver: Loaded
Stack plugin "o2cb": Loaded
Driver for "ocfs2_dlmfs": Loaded
Filesystem "ocfs2_dlmfs": Mounted
Checking O2CB cluster "ocfsstorage": Online
 Heartbeat dead threshold: 121
 Network idle timeout: 30000
 Network keepalive delay: 5000
 Network reconnect delay: 4000
 Heartbeat mode: Local
Checking O2CB heartbeat: Not active
Debug file system at /sys/kernel/debug: mounted

2.3 启动服务

systemctl enable o2cb.service
systemctl enable ocfs2.service
systemctl start o2cb.service
systemctl start ocfs2.service
systemctl status o2cb.service
systemctl status ocfs2.service

o2cb.init online
o2cb.init status

2.4 添加节点到集群

• 如果第二个节点启动失败，可以按这个步骤添加节点

mkdir -p /etc/ocfs2/
scp /etc/ocfs2/cluster.conf host-nfs-1722012841:/etc/ocfs2/
scp /etc/sysconfig/o2cb host-nfs-1722012841:/etc/sysconfig/
systemctl start o2cb.service
systemctl start ocfs2.service 
systemctl enable o2cb.service
systemctl enable ocfs2.service 
systemctl status o2cb.service
systemctl status ocfs2.service


more /etc/sysctl.conf
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
kernel.panic = 30
kernel.panic_on_oops = 1


o2cb.init status
o2cb.init online
systemctl enable o2cb.service
systemctl enable ocfs2.service 
systemctl status o2cb.service
systemctl status ocfs2.service

2.5 存储挂载

lsblk
fdisk /dev/sdb

# 一个节点执行即可
mkfs.ocfs2 --cluster-stack=o2cb -C 256K -J size=128M -N 16 -L ocfs2-disk1 --cluster-name=ocfsstorage --fs-feature-level=default -T vmstore /dev/sdb1


mkfs.ocfs2 1.8.6
Cluster stack: o2cb
Cluster name: ocfsstorage
Stack Flags: 0x0
NOTE: Feature extended slot map may be enabled
Filesystem Type of vmstore
Label: ocfs2-disk1
Features: sparse extended-slotmap backup-super unwritten inline-data stric
t-journal-super xattr indexed-dirs refcount discontig-bg
Block size: 4096 (12 bits)
Cluster size: 262144 (18 bits)
Volume size: 899999858688 (3433227 clusters) (219726528 blocks)
Cluster groups: 107 (tail covers 14091 clusters, rest cover 32256 cluster
s)
Extent allocator size: 171966464 (41 groups)
Journal size: 134217728
Node slots: 16
Creating bitmaps: done
Initializing superblock: done
Writing system files: done
Writing superblock: done
Writing backup superblock: 5 block(s)
Formatting Journals: done
Growing extent allocator: done
Formatting slot map: done
Formatting quota files: done
Writing lost+found: done
mkfs.ocfs2 successful

# 两个节点查看信息
mounted.ocfs2 -d

# 两台都要做

mkdir -p /opt/vms/disk1/
mount.ocfs2 /dev/sdb1 /opt/vms/disk1/

# 配置服务器启动⾃动挂载
vim /etc/rc.local
sleep 10
o2cb.init start
o2cb.init online
mount.ocfs2 /opt/vms/disk1/ /opt/vms/disk1/
chmod +x /etc/rc.local

3 nfs

yum install -y nfs-utils rpcbind
# Start and enable the NFS services
systemctl start nfs-server rpcbind
systemctl enable nfs-server rpcbind
systemctl restart nfs-server rpcbind

vi /etc/exports
/opt/vms/disk1 *(rw,sync,no_root_squash)

exportfs -r
exportfs -v
showmount -e
cat /proc/fs/nfsd/versions
cat /proc/fs/nfsd/versions
cat /etc/nfs.conf
cat /etc/nfsmount.conf
cat /etc/sysconfig/nfs

# 客户端连接测试
yum install -y nfs-utils rpcbind
mount -t nfs -o vers=4 172.20.128.41:/opt/vms/disk1 /tmp/test1

4 keepalived

yum install keepalived -y
systemctl start keepalived
systemctl enable keepalived
systemctl status keepalived

• 节点1 /etc/keepalived/keepalived.conf

  ! Configuration File for keepalived
  
  global_defs {
      router_id nfs1
      script_user root root
  }
  
  vrrp_script chk_nfs {
      script "/root/k_NFS_check.sh"
      interval 1
  }
  
  vrrp_instance VI_1 {
      state BACKUP
      interface ens192
      virtual_router_id 51
      priority 100
      advert_int 1
      nopreempt
      authentication {
          auth_type PASS
          auth_pass 123456
      }
      track_script {
          chk_nfs
      }
      virtual_ipaddress {
          172.20.128.42
      }
  }
  
  

• 节点2

  ! Configuration File for keepalived
  
  global_defs {
      router_id nfs2
      script_user root root
  }
  
  vrrp_script chk_nfs {
      script "/root/k_NFS_check.sh"
      interval 1
  }
  
  vrrp_instance VI_1 {
      state BACKUP
      interface ens192
      virtual_router_id 51
      priority 90
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass 123456
      }
      track_script {
          chk_nfs
      }
      virtual_ipaddress {
          172.20.128.42
      }
  }
  
  

  vi /root/k_NFS_check.sh
  
  
  
  #!/bin/bash
  
  # 检查 rpcbind 服务是否在运行，如果没有运行，则启动它
  A=$(ps -C rpcbind --no-header | wc -l)
  if [ "$A" -eq 0 ]; then
      systemctl start rpcbind
  fi
  
  # 检查 nfsd 服务是否在运行，如果没有运行，则启动它
  A=$(ps -C nfsd --no-header | wc -l)
  if [ "$A" -eq 0 ]; then
      systemctl start nfs
  fi
  
  # 检查 rpcbind 服务和 nfsd 服务的状态，如果任意一个服务未运行，则终止 keepalived
  if [ $(ps -C rpcbind --no-header | wc -l) -eq 0 ] || [ $(ps -C nfsd --no-header | wc -l) -eq 0 ]; then
      pkill keepalived
  fi 
  

    chmod u+x /root/k_NFS_check.sh
  

• 测试keepalived

  mount -t nfs -o vers=4 172.20.128.42:/opt/vms/disk1 /tmp/test1
  

  5 全测试

  systemctl status firewalld
  getenforce
  systemctl status o2cb.service
  systemctl status ocfs2.service
  o2cb.init status
  o2cb.init online
  systemctl status nfs-server
  systemctl status rpcbind
  systemctl status keepalived