整体架构
试验环境
1、操作系统 Centos7.9
主机 | 网卡 | 用途 |
---|---|---|
nginx1 | eth0:192.168.0.11 (vmnat8 nat模式) | 管理网络+对外服务 |
eth1:192.168.1.11(vmnat1 仅主机模式) | 模拟heartbeat心跳网络直连线 | |
VIP:192.168.0.15 | 对外服务 | |
nginx2 | eth0:192.168.0.11(vmnat8 nat模式) | 管理网络+对外服务 |
eth1:192.168.1.12(vmnat1 仅主机模式) | 模拟heartbeat心跳网络直连线 | |
VIP:192.168.0.15 | 对外服务 |
主机配置
# 关闭selinux
sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/selinux/config
setenforce 0
# 关闭防火墙
systemctl disable firewalld --now
# 配置主机名称
hostnamectl set-hostname heartbeat1
hostnamectl set-hostname heartbeat2
# 配置域名相互解析
echo "192.168.1.11 heartbeat1" >> /etc/hosts
echo "192.168.1.12 heartbeat2" >> /etc/hosts
# 下载源码包并上传到二台机器
http://www.linux-ha.org/wiki/Downloads
# 对应版本号
Heartbeat 3.0.6 Cluster Glue 1.0.12 Resource Agents 3.9.6
958e11be8686.tar.bz2 0a7add1d9996.tar.bz2 resource-agents-3.9.6.tar.gz
# 安装依赖包
yum install -y bzip2 autoconf automake libtool glib2-devel libxml2-devel bzip2-devel libtool-ltdl-devel asciidoc libuuid-devel net-tools
# 安装nginx
yum -y install nginx
源码部署
# 安装Cluster Glue
tar -jxvf 0a7add1d9996.tar.bz2
cd Reusable-Cluster-Components-glue--0a7add1d9996
groupadd haclient
useradd -g haclient hacluster
./autogen.sh
./configure --prefix=/usr/local/heartbeat --with-daemon-user=hacluster --with-daemon-group=haclient --enable-fatal-warnings=no LIBS='/lib64/libuuid.so.1'
make && make install
# 安装Resource Agents
tar -zxvf resource-agents-3.9.6.tar.gz
cd resource-agents-3.9.6/
./autogen.sh
./configure --prefix=/usr/local/heartbeat --with-daemon-user=hacluster --with-daemon-group=haclient --enable-fatal-warnings=no LIBS='/lib64/libuuid.so.1'
make && make install
# 安装heartbeat
tar xf 958e11be8686.tar.bz2
cd Heartbeat-3-0-958e11be8686/
./bootstrap
#添加环境变量
export CFLAGS="$CFLAGS -I/usr/local/heartbeat/include -L/usr/local/heartbeat/lib"
./configure --prefix=/usr/local/heartbeat --with-daemon-user=hacluster --with-daemon-group=haclient --enable-fatal-warnings=no LIBS='/lib64/libuuid.so.1'
make && make install
# 配置网卡支持插件文件
mkdir -pv /usr/local/heartbeat/usr/lib/ocf/lib/heartbeat/
cp /usr/lib/ocf/lib/heartbeat/ocf-* /usr/local/heartbeat/usr/lib/ocf/lib/heartbeat/
# 注意:一般启动时会报错因为 ping和ucast这些配置都需要插件支持 需要将lib64下面的插件软连接到lib目录 才不会抛出异常
ln -svf /usr/local/heartbeat/lib64/heartbeat/plugins/RAExec/* /usr/local/heartbeat/lib/heartbeat/plugins/RAExec/
ln -svf /usr/local/heartbeat/lib64/heartbeat/plugins/* /usr/local/heartbeat/lib/heartbeat/plugins/
# 拷贝配置文件
cp /usr/local/heartbeat/share/doc/heartbeat/{authkeys,haresources,ha.cf} /usr/local/heartbeat/etc/ha.d/
chmod 600 /usr/local/heartbeat/etc/ha.d/authkeys
配置heartbeat
heartbeat1配置
vim /usr/local/heartbeat/etc/ha.d/ha.cf
debugfile /var/log/ha-debug
logfile /var/log/ha-log
logfacility local0
keepalive 2
deadtime 30
warntime 10
initdead 120
udpport 694
ucast ens37 192.168.1.12
auto_failback on
node heartbeat1
node heartbeat2
heartbeat2配置
vim /usr/local/heartbeat/etc/ha.d/ha.cf
debugfile /var/log/ha-debug
logfile /var/log/ha-log
logfacility local0
keepalive 2
deadtime 30
warntime 10
initdead 120
udpport 694
ucast ens37 192.168.1.11
auto_failback on
node heartbeat1
node heartbeat2
配置authkeys配置文件
该文件表示发送心跳时,机器用于验证的key的hash算法,节点之间必须配置成一致的密码
vim /usr/local/heartbeat/etc/ha.d/authkeys
auth 3 #表示使用id为3的验证 下边需要定义一个3的验证算法
3 md5 Hello! #ID 3的验证加密为md5,并添加密码
配置haresources配置文件
二个节点设置成一样
vim /usr/local/heartbeat/etc/ha.d/haresources
heartbeat1 IPaddr::192.168.0.11/24/ens33 nginx
# 由于这里只是测试环境,就yum安装nginx了,脚本可以直接调用systemctl了,如果是二进制安装的自行编写脚本即可,要确保支持bash xxx start/stop这种形式
vim /etc/init.d/nginx
#!/bin/bash
case $1 in
start)
systemctl start nginx
;;
stop)
systemctl stop nginx
;;
esac
chmod a+x /etc/init.d/nginx
ln -s /etc/init.d/nginx /usr/local/heartbeat/etc/ha.d/resource.d/nginx
测试
# 双节点启动heartbeat
systemctl start heartbeat
heartbeat1信息
[root@heartbeat1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ca:06:8f brd ff:ff:ff:ff:ff:ff
inet 192.168.0.11/24 brd 192.168.0.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.0.15/24 brd 192.168.0.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::624c:c1db:e3b4:9165/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ca:06:99 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet6 fe80::bc35:d32b:9065:6bb0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
=====================================================================
[root@heartbeat1 ~]# systemctl status heartbeat.service
● heartbeat.service - Heartbeat High Availability Cluster Communication and Membership
Loaded: loaded (/usr/lib/systemd/system/heartbeat.service; disabled; vendor preset: disabled)
Active: active (running) since Wed 2021-08-11 05:35:40 EDT; 5min ago
Process: 47675 ExecStop=/usr/local/heartbeat/libexec/heartbeat/heartbeat -k (code=exited, status=0/SUCCESS)
Main PID: 47792 (heartbeat)
CGroup: /system.slice/heartbeat.service
├─47792 heartbeat: master control process
├─47795 heartbeat: FIFO reader
├─47796 heartbeat: write: ucast ens37
└─47797 heartbeat: read: ucast ens37
Aug 11 05:37:58 heartbeat1 heartbeat[47792]: [47792]: info: remote resource transition completed.
Aug 11 05:37:58 heartbeat1 heartbeat[47792]: [47792]: info: heartbeat1 wants to go standby [foreign]
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: info: standby: heartbeat2 can take our foreign resources
Aug 11 05:37:59 heartbeat1 heartbeat[48417]: [48417]: info: give up foreign HA resources (standby).
Aug 11 05:37:59 heartbeat1 heartbeat[48417]: [48417]: info: foreign HA resource release completed (standby).
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: info: Local standby process completed [foreign].
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: WARN: 1 lost packet(s) for [heartbeat2] [10:12]
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: info: remote resource transition completed.
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: info: No pkts missing from heartbeat2!
Aug 11 05:37:59 heartbeat1 heartbeat[47792]: [47792]: info: Other node completed standby takeover of foreign resources.
[root@heartbeat1 ~]# systemctl status nginx.service
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: active (running) since Wed 2021-08-11 05:36:28 EDT; 4min 34s ago
Process: 48102 ExecStart=/usr/sbin/nginx (code=exited, status=0/SUCCESS)
Process: 48099 ExecStartPre=/usr/sbin/nginx -t (code=exited, status=0/SUCCESS)
Process: 48097 ExecStartPre=/usr/bin/rm -f /run/nginx.pid (code=exited, status=0/SUCCESS)
Main PID: 48104 (nginx)
CGroup: /system.slice/nginx.service
├─48104 nginx: master process /usr/sbin/nginx
├─48105 nginx: worker process
└─48106 nginx: worker process
Aug 11 05:36:28 heartbeat1 systemd[1]: Starting The nginx HTTP and reverse proxy server...
Aug 11 05:36:28 heartbeat1 nginx[48099]: nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
Aug 11 05:36:28 heartbeat1 nginx[48099]: nginx: configuration file /etc/nginx/nginx.conf test is successful
Aug 11 05:36:28 heartbeat1 systemd[1]: Started The nginx HTTP and reverse proxy server
# 可以看出在启动hearbeat的时候,将nginx服务也起来了
将heartbeat1 heartbeat服务关闭
systemctl stop heartbeat
[root@heartbeat1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ca:06:8f brd ff:ff:ff:ff:ff:ff
inet 192.168.0.11/24 brd 192.168.0.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::624c:c1db:e3b4:9165/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:ca:06:99 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet6 fe80::bc35:d32b:9065:6bb0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@heartbeat1 ~]#
[root@heartbeat1 ~]# systemctl status nginx
● nginx.service - The nginx HTTP and reverse proxy server
Loaded: loaded (/usr/lib/systemd/system/nginx.service; disabled; vendor preset: disabled)
Active: inactive (dead)
Aug 11 05:32:28 heartbeat1 nginx[47230]: nginx: configuration file /etc/nginx/nginx.conf test is successful
Aug 11 05:32:28 heartbeat1 systemd[1]: Started The nginx HTTP and reverse proxy server.
Aug 11 05:32:36 heartbeat1 systemd[1]: Stopping The nginx HTTP and reverse proxy server...
Aug 11 05:32:36 heartbeat1 systemd[1]: Stopped The nginx HTTP and reverse proxy server.
Aug 11 05:36:28 heartbeat1 systemd[1]: Starting The nginx HTTP and reverse proxy server...
Aug 11 05:36:28 heartbeat1 nginx[48099]: nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
Aug 11 05:36:28 heartbeat1 nginx[48099]: nginx: configuration file /etc/nginx/nginx.conf test is successful
Aug 11 05:36:28 heartbeat1 systemd[1]: Started The nginx HTTP and reverse proxy server.
Aug 11 05:43:45 heartbeat1 systemd[1]: Stopping The nginx HTTP and reverse proxy server...
Aug 11 05:43:45 heartbeat1 systemd[1]: Stopped The nginx HTTP and reverse proxy server.
# 可以发现nginx也停了
heartbeat2信息
[root@heartbeat2 ha.d]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:91:a8:8e brd ff:ff:ff:ff:ff:ff
inet 192.168.0.12/24 brd 192.168.0.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.0.15/24 brd 192.168.0.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::624c:c1db:e3b4:9165/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::2e8f:238e:c202:4027/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:91:a8:98 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.12/24 brd 192.168.1.255 scope global noprefixroute ens37
valid_lft forever preferred_lft forever
inet6 fe80::bc35:d32b:9065:6bb0/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::38dc:b558:31b0:8fda/64 scope link noprefixroute
valid_lft forever preferred_lft forever