Haproxy+Keepalived实现RabbtMQ Cluster高可用

系统架构

                                          |
                                  [VIP:11.1.1.100]	
+---------+-------------------------------+-------------------------------+----------+
          |                               |                               |
+---------+------------+                  |                   +----------------------+
| [  HAProxy Node #1 ] |                  |                   | [  HAProxy Node #2 ] |
|    ha1.sumail.com    |                  |                   |    ha2.sumail.com    |
|      11.1.1.61       |                  |                   |      11.1.1.62       |
+----------------------+                  |                   +----------------------+
                                          |                                
+---------+-------------------------------+-------------------------------+----------+
          |                               |                               |
+---------+------------+       +----------+-----------+       +-----------+----------+
| [  Cluster Node #1 ] |       | [  Cluster Node #2 ] |       | [  Cluster Node #3 ] |
|   node1.sumail.com   +<------+   node2.sumail.com   +<------+   node3.sumail.com   |
|      11.1.1.51       |       |      11.1.1.52       |       |      11.1.1.53       |
+----------------------+       +----------------------+       +----------------------+

系统环境配置

[root@ha1 ~]# vi /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

11.1.1.51   node1 node1.sumail.com
11.1.1.52   node2 node2.sumail.com
11.1.1.53   node3 node3.sumail.com

11.1.1.61   ha1   ha1.sumail.com
11.1.1.62   ha2   ha2.sumail.com                             


[root@ha1 ~]# scp /etc/hosts ha2:/etc/hosts
The authenticity of host 'ha2 (11.1.1.62)' can't be established.
ED25519 key fingerprint is SHA256:BQD837RdlkpurpiYQYlLVeWNmk1zRFwIjeHo7YnMcfk.
This key is not known by any other names
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added 'ha2' (ED25519) to the list of known hosts.
root@ha2's password: 
hosts                                                                                                                                                                          100%  331   322.6KB/s   00:00    
[root@ha1 ~]# scp /etc/hosts node1:/etc/hosts
The authenticity of host 'node1 (11.1.1.51)' can't be established.
ED25519 key fingerprint is SHA256:BQD837RdlkpurpiYQYlLVeWNmk1zRFwIjeHo7YnMcfk.
This host key is known by the following other names/addresses:
    ~/.ssh/known_hosts:1: ha2
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added 'node1' (ED25519) to the list of known hosts.
root@node1's password: 
hosts                                                                                                                                                                          100%  331   116.5KB/s   00:00    
[root@ha1 ~]# scp /etc/hosts node2:/etc/hosts
The authenticity of host 'node2 (11.1.1.52)' can't be established.
ED25519 key fingerprint is SHA256:BQD837RdlkpurpiYQYlLVeWNmk1zRFwIjeHo7YnMcfk.
This host key is known by the following other names/addresses:
    ~/.ssh/known_hosts:1: ha2
    ~/.ssh/known_hosts:4: node1
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added 'node2' (ED25519) to the list of known hosts.
root@node2's password: 
hosts                                                                                                                                                                          100%  331   248.3KB/s   00:00    
[root@ha1 ~]# scp /etc/hosts node3:/etc/hosts
The authenticity of host 'node3 (11.1.1.53)' can't be established.
ED25519 key fingerprint is SHA256:BQD837RdlkpurpiYQYlLVeWNmk1zRFwIjeHo7YnMcfk.
This host key is known by the following other names/addresses:
    ~/.ssh/known_hosts:1: ha2
    ~/.ssh/known_hosts:4: node1
    ~/.ssh/known_hosts:5: node2
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added 'node3' (ED25519) to the list of known hosts.
root@node3's password: 
hosts                                                                                                                                                                          100%  331   201.0KB/s   00:00    

RabbitMQ Cluster 安装与配置

安装Haproxy、Keepalived

# ha1
[root@ha1 ~]# dnf install haproxy keepalived -y

# ha2
[root@ha2 ~]# dnf install haproxy keepalived -y

编辑日志配置文件

# 编辑rsyslog配置文件
[root@ha1 ~]# vi /etc/rsyslog.conf

# 取消30行和31行前的注释
module(load="imudp") # needs to be done just once
input(type="imudp" port="514")
# 在32行添加如下内容
$AllowedSender  UDP, 127.0.0.1
 
 # 修改47行
*.info;mail.none;authpriv.none;cron.none;local2.none               /var/log/messages
# 添加48行内容
local2.*                                                           /var/log/haproxy.log


# 复制配置文件到ha2
[root@ha1 ~]# scp /etc/rsyslog.conf ha2:/etc/rsyslog.conf 
root@ha2's password: 
rsyslog.conf 
# 重启rsyslog
[root@ha1 ~]# systemctl restart rsyslog

[root@ha2 ~]# systemctl restart rsyslog

配置Haproxy

# haproxy 配置参考文档:https://docs.haproxy.org/
# 备份配置文件
[root@ha1 ~]# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

[root@ha1 ~]# vim /etc/haproxy/haproxy.cfg  
global
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     2000000
    user        haproxy
    group       haproxy
    daemon

    # nbthread  # 指定单个haproxy进程开启的线程数  启用此参数不能开启多核  

    # nbproc 4  #开启的haproxy进程数,建议与CPU核数保持一致
    # cpu-map 1 0
    # cpu-map 2 1
    # cpu-map 3 2
    # cpu-map 4 3
    # stats bind-process 4
    stats socket /var/lib/haproxy/stats

    ssl-default-bind-ciphers PROFILE=SYSTEM
    ssl-default-server-ciphers PROFILE=SYSTEM

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    #maxconn                 3000
# 监控页面
listen monitor-stats
    bind *:80
    option      forwardfor
    stats       enable
    stats       auth    admin:admin
    stats       hide-version
    stats       show-node
    stats       refresh 60s
    stats       uri     /haproxy?stats

# rabbitmq api接口
listen mq_cluster_5672
    bind *:5672
    mode tcp
    balance     roundrobin
    server      node1   11.1.1.51:5672  check
    server      node2   11.1.1.52:5672  check
    server      node3   11.1.1.53:5672  check
    # check             #       检测参数 默认不开启
    # addr      IP              #       指定待检测的IP
    # port      num             #       指定待检测的端口
    # inter     num             #       检测周期  默认2s
    # fall      num             #       后端服务器检测失败次数  默认3次
    # rise      num             #       后端服务器从下线到恢复检查次数，默认为2
    # weight    num     #       默认为1，最大值为256,0表示不参与负载均衡
    # backup            #       将石端服务器标记为备份认态
    # disable           #       将后端吸务器标记为不可丹状态

# rabbitmq 管理界面
listen mq_admin_15672
    bind *:15672
    mode tcp
    balance     roundrobin
    server      node1   11.1.1.51:15672 check
    server      node2   11.1.1.52:15672 check
    server      node3   11.1.1.53:15672 check


[root@ha1 ~]# scp /etc/haproxy/haproxy.cfg ha2:/etc/haproxy/haproxy.cfg
root@ha2's password: 
haproxy.cfg 
    
    
# 启动服务
[root@ha1 ~]# systemctl enable haproxy --now
Created symlink /etc/systemd/system/multi-user.target.wants/haproxy.service → /usr/lib/systemd/system/haproxy.service.

[root@ha2 ~]# systemctl enable haproxy --now
Created symlink /etc/systemd/system/multi-user.target.wants/haproxy.service → /usr/lib/systemd/system/haproxy.service.

防火墙放行端口

# 放行端口
[root@ha1 ~]# firewall-cmd --add-service=http
success
[root@ha1 ~]# firewall-cmd --add-port=15672/tcp
success
[root@ha1 ~]# firewall-cmd --add-port=5672/tcp
success
[root@ha1 ~]# firewall-cmd --runtime-to-permanent
success

[root@node1 ~] firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept'
[root@node1 ~] firewall-cmd --runtime-to-permanent

• 查看haproxy

node1

连续刷新

node2

配置Keepalived

编辑配置文件

[root@ha1 ~]# vim /etc/keepalived/keepalived.conf
global_defs {
   router_id HA1
}

# vrrp 检测haproxy状态脚本
vrrp_script chk_haproxy {
    script "/etc/keepalived/scripts/haproxy_check.sh"
    interval 2
    timeout 2
    fall 3
}

# VRRP配置
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

    track_script {
        chk_haproxy
    }

    virtual_ipaddress {
        11.1.1.100
    }
}

# haproxy 检测脚本
[root@ha1 ~]# vim /etc/keepalived/scripts/haproxy_check.sh
#!/bin/bash
CMD = `ps -C haproxy --no-header | wc -l`

if [ $CMD -eq 0 ]; then
        systemctl start haproxy
        sleep 2
        if [ $CMD -eq 0 ]; then
                killall keepalived
        fi
fi

[root@ha2 ~]# mkdir /etc/keepalived/scripts
[root@ha1 ~]# scp /etc/keepalived/keepalived.conf ha2:/etc/keepalived/keepalived.conf
root@ha2's password: 
keepalived.conf                                                                                                                                                                                                                             100%  432   385.8KB/s   00:00    
[root@ha1 ~]# scp /etc/keepalived/scripts/haproxy_check.sh  ha2:/etc/keepalived/scripts/haproxy_check.sh 
root@ha2's password: 
haproxy_check.sh                                                                                                                                                                                                                            100%  188   240.9KB/s   00:00    
[root@ha1 ~]# systemctl enable keepalived --now
[root@ha1 ~]# firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept'
[root@ha1 ~]# firewall-cmd --runtime-to-permanent

[root@ha2 ~]# systemctl enable keepalived --now
[root@ha2 ~]# firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept'
[root@ha2 ~]# firewall-cmd --runtime-to-permanent


[root@ha1 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:50:56:3a:36:d2 brd ff:ff:ff:ff:ff:ff
    altname enp2s1
    inet 11.1.1.61/24 brd 11.1.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 11.1.1.100/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:fe3a:36d2/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@ha1 ~]# 

[root@ha1 ~]# systemctl stop keepalived
[root@ha1 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:50:56:3a:36:d2 brd ff:ff:ff:ff:ff:ff
    altname enp2s1
    inet 11.1.1.61/24 brd 11.1.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:fe3a:36d2/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever


[root@ha2 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:50:56:27:d3:3e brd ff:ff:ff:ff:ff:ff
    altname enp2s1
    inet 11.1.1.62/24 brd 11.1.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 11.1.1.100/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:fe27:d33e/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

刷新页面

[root@ha1 ~]# systemctl start keepalived
[root@ha1 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:50:56:3a:36:d2 brd ff:ff:ff:ff:ff:ff
    altname enp2s1
    inet 11.1.1.61/24 brd 11.1.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 11.1.1.100/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::250:56ff:fe3a:36d2/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever