Docker----rabbitmq集群高可用（3节点）

一、环境准备

准备三个docker环境

docker-machine create -d "virtualbox" docker1
docker-machine create -d "virtualbox" docker2
docker-machine create -d "virtualbox" docker3

主机名	IP
docker1	192.168.99.100
docker2	192.168.99.101
docker3	192.168.99.102

登录到各个docker

docker-machine ssh docker1
docker-machine ssh docker2
docker-machine ssh docker3

二、rabbitmq集群搭建

1、使用docker启动三个rabbitmq服务

# docker1
docker run -d -h mqnode1 --name mqnode1 \
--add-host mqnode2:192.168.99.101 \
--add-host mqnode3:192.168.99.102 \
-p4369:4369 -p5671-5672:5671-5672 -p15671-15672:15671-15672 -p 25672:25672 \
-e RABBITMQ_NODENAME=rabbitmq@mqnode1 \
-e RABBITMQ_ERLANG_COOKIE='mqcluster' \
-e RABBITMQ_DEFAULT_USER=admin \
-e RABBITMQ_DEFAULT_PASS=admin \
rabbitmq:3.7-management

# docker2
docker run -d -h mqnode2 --name mqnode2 \
--add-host mqnode1:192.168.99.100 \
--add-host mqnode3:192.168.99.102 \
-p4369:4369 -p5671-5672:5671-5672 -p15671-15672:15671-15672 -p 25672:25672 \
-e RABBITMQ_NODENAME=rabbitmq@mqnode2 \
-e RABBITMQ_ERLANG_COOKIE='mqcluster' \
-e RABBITMQ_DEFAULT_USER=admin \
-e RABBITMQ_DEFAULT_PASS=admin \
rabbitmq:3.7-management

# docker3
docker run -d -h mqnode3 --name mqnode3 \
--add-host mqnode1:192.168.99.100 \
--add-host mqnode2:192.168.99.101 \
-p4369:4369 -p5671-5672:5671-5672 -p15671-15672:15671-15672 -p 25672:25672 \
-e RABBITMQ_NODENAME=rabbitmq@mqnode3 \
-e RABBITMQ_ERLANG_COOKIE='mqcluster' \
-e RABBITMQ_DEFAULT_USER=admin \
-e RABBITMQ_DEFAULT_PASS=admin \
rabbitmq:3.7-management

2、使其余两个节点加入集群

–ram 设置rabbitmq为内存存储模式
–disc 设置rabbitmq为硬盘存储模式
使mqnode2加入rabbitmq@mqnode1节点

docker@docker2:~$ docker exec -it mqnode2 bash
root@mqnode2:/# rabbitmqctl stop_app
Stopping rabbit application on node rabbitmq@mqnode2 ...
root@mqnode2:/# rabbitmqctl reset
Resetting node rabbitmq@mqnode2 ...
root@mqnode2:/# rabbitmqctl join_cluster --ram rabbitmq@mqnode1
Clustering node rabbitmq@mqnode2 with rabbitmq@mqnode1
root@mqnode2:/# rabbitmqctl start_app
Starting node rabbitmq@mqnode2 ...
 completed with 3 plugins

查看mqnode1集群状态

docker@docker1:~$ docker exec -it mqnode1 bash
root@mqnode1:/# rabbitmqctl cluster_status
Cluster status of node rabbitmq@mqnode1 ...
[{nodes,[{disc,[rabbitmq@mqnode1,rabbitmq@mqnode2]}]},
 {running_nodes,[rabbitmq@mqnode2,rabbitmq@mqnode1]},
 {cluster_name,<<"rabbitmq@mqnode1">>},
 {partitions,[]},
 {alarms,[{rabbitmq@mqnode2,[]},{rabbitmq@mqnode1,[]}]}]

使mqnode3加入rabbitmq@mqnode1节点

docker@docker3:~$ docker exec -it mqnode3 bash
root@mqnode3:/# rabbitmqctl stop_app
Stopping rabbit application on node rabbitmq@mqnode3 ...
root@mqnode3:/# rabbitmqctl reset
Resetting node rabbitmq@mqnode3 ...
root@mqnode3:/# rabbitmqctl join_cluster --ram rabbitmq@mqnode1
Clustering node rabbitmq@mqnode3 with rabbitmq@mqnode1
root@mqnode3:/# rabbitmqctl start_app
Starting node rabbitmq@mqnode3 ...
 completed with 3 plugins.

查看mqnode1集群状态

root@mqnode1:/# rabbitmqctl cluster_status
Cluster status of node rabbitmq@mqnode1 ...
[{nodes,[{disc,[rabbitmq@mqnode1,rabbitmq@mqnode2,rabbitmq@mqnode3]}]},
 {running_nodes,[rabbitmq@mqnode3,rabbitmq@mqnode2,rabbitmq@mqnode1]},
 {cluster_name,<<"rabbitmq@mqnode1">>},
 {partitions,[]},
 {alarms,[{rabbitmq@mqnode3,[]},{rabbitmq@mqnode2,[]},{rabbitmq@mqnode1,[]}]}]

登录mqnode1管理控制台查看集群状态：http://192.168.99.100:15672

3、配置镜像队列

按如下操作，查看队列显示+2即可

docker@docker1:~$ docker exec -it mqnode1 bash
root@mqnode1:/# rabbitmqctl set_policy ha-all "^" '{"ha-mode":"all"}'
root@mqnode1:/# rabbitmqctl set_permissions -p / admin ".*" ".*" ".*"
root@mqnode1:/# rabbitmqadmin -H192.168.99.100 -P15672 -uadmin -padmin declare queue --vhost=/ name=q.test durable=true

至此rabbitmq集群搭建完成。

三、实现高可用（二选一）

1、配置nginx负载均衡

启动nginx，4672端口用于反向代理rabbitmq集群的5672端口，6672端口用于反向代理rabbitmq集群的15672端口

# docker3
docker run -d --name nginx -p 80:80 -p 443:443 -p 4672:4672 -p 6672:6672 \
-v /home/docker/nginx/:/etc/nginx \
nginx

vi /etc/nginx/nginx.conf配置stream模块

.....
events {
    worker_connections  1024;
}

stream {
    upstream mq5672 {                      
        server 192.168.99.100:5672;             
        server 192.168.99.101:5672;             
        server 192.168.99.102:5672;                                               
    }
    upstream mq15672 {
        server 192.168.99.100:15672;
        server 192.168.99.101:15672;
        server 192.168.99.102:15672;
    }
    server {
        listen 4672;
        proxy_pass mq5672;
    }
    server {
        listen 6672;
        proxy_pass mq15672;                   
    }
}
http {
.....

登录管理控制台界面访问：http://192.168.99.102:6672/
程序连接：192.168.99.102:4672

2、配置haproxy负载均衡

haproxy.cfg配置

global #全局属性
    maxconn 256  #最大同时256连接
    user root
    group root
    pidfile /var/run/haproxy.pid
    log 127.0.0.1 local0 info
    log 127.0.0.1 local1 warning


defaults #默认参数
    log global
    mode http
    option httplog
    option dontlognull
    option tcplog
    timeout connect 5000ms  #连接server端超时5s
    timeout client 10000ms  #客户端响应超时10s
    timeout server 10000ms  #server端响应超时10s

# rabbitmq服务TCP代理
listen rabbitmq
    bind *:4672
    mode tcp # 4层代理
    balance roundrobin
    server mqnode1 mqnode1:5672 weight 1 maxconn 60000 check inter 3s
    server mqnode2 mqnode2:5672 weight 1 maxconn 60000 check inter 3s
    server mqnode3 mqnode3:5672 weight 1 maxconn 60000 check inter 3s


# rabbitmq-management服务HTTP代理
frontend rabbitmq-management
    bind *:6672
    mode http
    default_backend rabbitmq-management

# rabbitmq-management backend
backend rabbitmq-management
    balance roundrobin
    option forwardfor
    server mqnode1-management mqnode1:15672 check
    server mqnode2-management mqnode2:15672 check
    server mqnode3-management mqnode3:15672 check

启动haproxy

# docker2
docker run -d --name mqhaproxy \
-p 4672:4672 -p 6672:6672 \
--add-host mqnode1:192.168.99.100 \
--add-host mqnode2:192.168.99.101 \
--add-host mqnode3:192.168.99.102 \
-v /home/docker/haproxy/:/usr/local/etc/haproxy:ro \
haproxy:2.0

登录管理控制台界面访问：http://192.168.99.101:6672/
程序连接：192.168.99.101:4672