Docker 系列之 docker-compose 部署 Nacos 集群（使用 Keepalived 实现主备集群）

docker-compose 部署 三节点，多机 + Keepalived

一、机器规划

准备三台机器，在每台机器上，计划各安装一个 nacos-server

机器IP	节点	节点端口
192.168.9.60	nacos-server	8848
192.168.9.61	nacos-server	8848
192.168.9.62	nacos-server	8848

二、nacos 集群准备工作

1、在宿主机上创建 conf 和 logs 文件夹，我这里创建在 /data/nacos 目录下

mkdir -p /data/nacos/conf
mkdir -p /data/nacos/logs

nacos子文件夹	作用	包含文件
conf	用于存放nacos配置文件	application.properties、cluster.conf
logs	用于存放nacos日志文件

2、修改 nacos 配置文件

application.properties 配置文件内容如下，放入三台机器的 conf 文件夹中

# spring
server.servlet.contextPath=${SERVER_SERVLET_CONTEXTPATH:/nacos}
server.contextPath=/nacos
server.port=${NACOS_APPLICATION_PORT:8848}
server.tomcat.accesslog.max-days=30
server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D %{User-Agent}i %{Request-Source}i
spring.datasource.platform=${SPRING_DATASOURCE_PLATFORM:mysql}
nacos.cmdb.dumpTaskInterval=3600
nacos.cmdb.eventTaskInterval=10
nacos.cmdb.labelTaskInterval=300
nacos.cmdb.loadDataAtStart=false
db.num=${MYSQL_DATABASE_NUM:1}
db.url.0=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?${MYSQL_SERVICE_DB_PARAM:characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false}
db.url.1=jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT:3306}/${MYSQL_SERVICE_DB_NAME}?${MYSQL_SERVICE_DB_PARAM:characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false}
db.user=${MYSQL_SERVICE_USER}
db.password=${MYSQL_SERVICE_PASSWORD}
### The auth system to use, currently only 'nacos' and 'ldap' is supported:
nacos.core.auth.system.type=${NACOS_AUTH_SYSTEM_TYPE:nacos}
### worked when nacos.core.auth.system.type=nacos
### The token expiration in seconds:
nacos.core.auth.plugin.nacos.token.expire.seconds=${NACOS_AUTH_TOKEN_EXPIRE_SECONDS:18000}
### The default token:
nacos.core.auth.plugin.nacos.token.secret.key=${NACOS_AUTH_TOKEN:SecretKey012345678901234567890123456789012345678901234567890123456789}
### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay.
nacos.core.auth.enabled=true
nacos.core.auth.caching.enabled=${NACOS_AUTH_CACHE_ENABLE:false}
nacos.core.auth.enable.userAgentAuthWhite=${NACOS_AUTH_USER_AGENT_AUTH_WHITE_ENABLE:false}
nacos.core.auth.server.identity.key=${NACOS_AUTH_IDENTITY_KEY:serverIdentity}
nacos.core.auth.server.identity.value=${NACOS_AUTH_IDENTITY_VALUE:security}
server.tomcat.accesslog.enabled=${TOMCAT_ACCESSLOG_ENABLED:false}
# default current work dir
server.tomcat.basedir=file:.
## spring security config
### turn off security
nacos.security.ignore.urls=${NACOS_SECURITY_IGNORE_URLS:/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-fe/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/**}
# metrics for elastic search
management.metrics.export.elastic.enabled=false
management.metrics.export.influx.enabled=false
nacos.naming.distro.taskDispatchThreadCount=10
nacos.naming.distro.taskDispatchPeriod=200
nacos.naming.distro.batchSyncKeyCount=1000
nacos.naming.distro.initDataRatio=0.9
nacos.naming.distro.syncRetryDelay=5000
nacos.naming.data.warmup=true

cluster.conf 配置文件内容如下，放入三台机器的 conf 文件夹中

# ip:port
192.168.9.60:8848
192.168.9.61:8848
192.168.9.62:8848

3、编辑 nacos 的 docker-compose.yml 文件

在三台机器上创建 docker-compose.yml 文件

• nacos01

version: '3'
services:
  nacos01:
    image: nacos/nacos-server:v2.2.3 #指定nacos镜像版本，如果docker开始没下载，这里会自动下载并创建容器
    container_name: nacos01
    hostname: nacos01  #这里设置主机名，是方便在同一个服务器docker环境中各个nacos服务能发现对方
    restart: always
    ports:
      - "8848:8848"  # 宿主机端口：映射容器端口
    environment:
      NACOS_SERVERS: nacos01:8848 nacos02:8848 nacos03:8848 # IP1,IP2是对应的服务器外网,PORT1,PORT2是对应的nacos服务端口
      MODE: cluster # 集群模式
      BASE_DIR: /home/nacos
      NACOS_DEBUG: y
      TIME_ZONE: Asia/Shanghai
      PREFER_HOST_MODE: hostname
      SPRING_DATASOURCE_PLATFORM: mysql
      MYSQL_SERVICE_DB_NAME: nacos
      MYSQL_SERVICE_HOST: # mysql地址
      MYSQL_SERVICE_PORT: # mysql端口
      MYSQL_SERVICE_USER: # mysql用户名
      MYSQL_SERVICE_PASSWORD: # mysql密码
      MYSQL_SERVICE_DB_PARAM: useSSL=false&characterEncoding=utf8&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
      MYSQL_DATABASE_NUM: 1
      NACOS_AUTH_TOKEN: SecretKey012345678901234567890123456789012345678901234567890123456789
      NACOS_AUTH_IDENTITY_KEY: nacos_identity_key
      NACOS_AUTH_IDENTITY_VALUE: nacos_identity_value
    network_mode: host
    extra_hosts:
      - "nacos01:192.168.9.60"
      - "nacos02:192.168.9.61"
      - "nacos03:192.168.9.62"
    volumes: #宿主机地址或文件：映射容器地址和文件
      - /etc/localtime:/etc/localtime
      - /data/nacos/logs:/home/nacos/logs  #存放nacos日志
      - /data/nacos/conf/application.properties:/home/nacos/conf/application.properties #存放nacos配置文件
      - /data/nacos/conf/cluster.conf:/home/nacos/conf/cluster.conf #存放nacos配置文件
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G
        reservations:
          cpus: '2'
          memory: 4G

• nacos02

version: '3'
services:
  nacos02:
    image: nacos/nacos-server:v2.2.3 #指定nacos镜像版本，如果docker开始没下载，这里会自动下载并创建容器
    container_name: nacos02
    hostname: nacos02  #这里设置主机名，是方便在同一个服务器docker环境中各个nacos服务能发现对方
    restart: always
    ports:
      - "8848:8848"  # 宿主机端口：映射容器端口
    environment:
      NACOS_SERVERS: nacos01:8848 nacos02:8848 nacos03:8848 # IP1,IP2是对应的服务器外网,PORT1,PORT2是对应的nacos服务端口
      MODE: cluster # 集群模式
      BASE_DIR: /home/nacos
      NACOS_DEBUG: y
      TIME_ZONE: Asia/Shanghai
      PREFER_HOST_MODE: hostname
      SPRING_DATASOURCE_PLATFORM: mysql
      MYSQL_SERVICE_DB_NAME: nacos
      MYSQL_SERVICE_HOST: # mysql地址
      MYSQL_SERVICE_PORT: # mysql端口
      MYSQL_SERVICE_USER: # mysql用户名
      MYSQL_SERVICE_PASSWORD: # mysql密码
      MYSQL_SERVICE_DB_PARAM: useSSL=false&characterEncoding=utf8&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
      MYSQL_DATABASE_NUM: 1
      NACOS_AUTH_TOKEN: SecretKey012345678901234567890123456789012345678901234567890123456789
      NACOS_AUTH_IDENTITY_KEY: nacos_identity_key
      NACOS_AUTH_IDENTITY_VALUE: nacos_identity_value
    network_mode: host
    extra_hosts:
      - "nacos01:192.168.9.60"
      - "nacos02:192.168.9.61"
      - "nacos03:192.168.9.62"
    volumes: #宿主机地址或文件：映射容器地址和文件
      - /etc/localtime:/etc/localtime
      - /data/nacos/logs:/home/nacos/logs  #存放nacos日志
      - /data/nacos/conf/application.properties:/home/nacos/conf/application.properties #存放nacos配置文件
      - /data/nacos/conf/cluster.conf:/home/nacos/conf/cluster.conf #存放nacos配置文件
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G
        reservations:
          cpus: '2'
          memory: 4G

• nacos03

version: '3'
services:
  nacos03:
    image: nacos/nacos-server:v2.2.3 #指定nacos镜像版本，如果docker开始没下载，这里会自动下载并创建容器
    container_name: nacos03
    hostname: nacos03  #这里设置主机名，是方便在同一个服务器docker环境中各个nacos服务能发现对方
    restart: always
    ports:
      - "8848:8848"  # 宿主机端口：映射容器端口
    environment:
      NACOS_SERVERS: nacos01:8848 nacos02:8848 nacos03:8848 # IP1,IP2是对应的服务器外网,PORT1,PORT2是对应的nacos服务端口
      MODE: cluster # 集群模式
      BASE_DIR: /home/nacos
      NACOS_DEBUG: y
      TIME_ZONE: Asia/Shanghai
      PREFER_HOST_MODE: hostname
      SPRING_DATASOURCE_PLATFORM: mysql
      MYSQL_SERVICE_DB_NAME: nacos
      MYSQL_SERVICE_HOST: # mysql地址
      MYSQL_SERVICE_PORT: # mysql端口
      MYSQL_SERVICE_USER: # mysql用户名
      MYSQL_SERVICE_PASSWORD: # mysql密码
      MYSQL_SERVICE_DB_PARAM: useSSL=false&characterEncoding=utf8&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
      MYSQL_DATABASE_NUM: 1
      NACOS_AUTH_TOKEN: SecretKey012345678901234567890123456789012345678901234567890123456789
      NACOS_AUTH_IDENTITY_KEY: nacos_identity_key
      NACOS_AUTH_IDENTITY_VALUE: nacos_identity_value
    network_mode: host
    extra_hosts:
      - "nacos01:192.168.9.60"
      - "nacos02:192.168.9.61"
      - "nacos03:192.168.9.62"
    volumes: #宿主机地址或文件：映射容器地址和文件
      - /etc/localtime:/etc/localtime
      - /data/nacos/logs:/home/nacos/logs  #存放nacos日志
      - /data/nacos/conf/application.properties:/home/nacos/conf/application.properties #存放nacos配置文件
      - /data/nacos/conf/cluster.conf:/home/nacos/conf/cluster.conf #存放nacos配置文件
    deploy:
      resources:
        limits:
          cpus: '4'
          memory: 8G
        reservations:
          cpus: '2'
          memory: 4G

三、 启动 nacos 服务

在三台机器的 /etc/hosts 文件中加上下面 yml 配置中的 hostname

echo "192.168.9.60 nacos01
192.168.9.61 nacos02
192.168.9.62 nacos03" >> /etc/hosts

分别在三台机器上执行启动命令

# 启动命令
docker-compose -f docker-compose.yml up -d

如图所示，代表启动成功

四、安装 keepalived 并设置 VIP

安装keepalived

yum install -y keepalived;
#设置开机启动
systemctl enable keepalived
cp /usr/sbin/keepalived /etc/init.d/keepalived
chmod +x /etc/init.d/keepalived

check_proxy_nacos.sh

当 doris 挂掉，则返回1，keepalived 权重降低，主节点被抢占，让另外一台机器接手

vi /etc/keepalived/check_proxy_nacos.sh

#!/bin/bash
counter=$(netstat -na|grep "LISTEN"|grep "8848"|wc -l)
if [ "${counter}" -eq 0 ]; then
  exit 1
else
  exit 0
fi

确保这个脚本有执行权限，并且正确地配置了监听端口。如果监控的服务端口不可用，脚本将返回非零值，这将导致Keepalived认为本地服务器失效，并开始VIP转移的过程。

chmod +x /etc/keepalived/check_proxy_nacos.sh

keepalived.conf

编辑 keepalived 配置文件 /etc/keepalived/keepalived.conf

参考资料：https://blog.csdn.net/cnskylee/article/details/131591733

vi /etc/keepalived/keepalived.conf

• mater机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha01
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_nacos.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_3 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 68
   # 在ha2上为80、h3上为60
   priority 100
   # 本机ip
   unicast_src_ip 192.168.9.60
   unicast_peer {   
       #对端ip
       192.168.9.61
       192.168.9.62
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        10.250.81.246 dev bond0 label bond0:3
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_nacos
   }
}

• node1机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha02
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_nacos.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_3 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 68
   # 在ha2上为80、h3上为60
   priority 80
   # 本机ip
   unicast_src_ip 192.168.9.61
   unicast_peer {   
       #对端ip
       192.168.9.60
       192.168.9.62
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        10.250.81.246 dev bond0 label bond0:3
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_nacos
   }
}

• node2机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha03
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_nacos.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_3 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 68
   # 在ha2上为80、h3上为60
   priority 60
   # 本机ip
   unicast_src_ip 192.168.9.62
   unicast_peer {   
       #对端ip
       192.168.9.61
       192.168.9.60
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        10.250.81.246 dev bond0 label bond0:3
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_nacos
   }
}

启动keepalived

systemctl start keepalived

查看keepalived状态

systemctl status keepalived

使用 VIP 访问服务

参考资料

https://nacos.io/docs/v2.3/guide/admin/cluster-mode-quick-start/
https://blog.csdn.net/xingzuo_1840/article/details/137588589