1.集群规划
| IP | 类型 | 机房 | 说明 |
|---|---|---|---|
| 10.242.178.130 | consul server | 新港 | Consul v1.10.2 |
| 10.242.178.131 | consul server | 新港 | Consul v1.10.2 |
| 10.242.178.129 | consul server | 新港 | Consul v1.10.2 |
| 10.243.204.103 | consul client | 新港 | Consul v1.10.2 |
2.安装部署
2.1.文件系统规划
-
consul server
lvcreate -n consullv datavg -L 1G mkdir /consul mkfs.ext3 /dev/datavg/consullv echo "/dev/datavg/consullv /consul ext3 defaults,noatime,nodiratime 0 0" >>/etc/fstab mount -a mkdir /consul/data mkdir /consul/etc mkdir /consul/bin mkdir /consul/log
-
consul client
mkdir -p /opt/consul/data mkdir -p /opt/consul/etc mkdir -p /opt/consul/bin mkdir -p /opt/consul/log
2.2.软件部署
将consul直接复制至/consul/bin(server)和/opt/consul/bin(client)下即可
rsync -e "sudo ssh" consul 10.242.178.130:/consul/bin/ rsync -e "sudo ssh" consul 10.242.178.131:/consul/bin/ rsync -e "sudo ssh" consul 10.242.178.129:/consul/bin/ rsync -e "sudo ssh" consul 10.243.204.103:/opt/consul/bin/
2.3.配置文件
-
server 10.242.178.130 /consul/etc/server.json 首次启动需注册tokens.default并bootstrap_expect设置为1
{
"data_dir": "/consul/data",
"datacenter": "xingang",
"primary_datacenter":"xingang",
"log_level": "INFO",
"server": true,
"bootstrap_expect": 3,
"bind_addr": "10.242.178.130",
"client_addr": "0.0.0.0",
"recursors":["10.237.255.253"],
"start_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"retry_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"node_name": "dnsservernode1",
"encrypt": "PwGyXsuuc6w0Jx6noxi6PwrN6GeH1HHYBpZ4E77mEwk=",
"log_file":"/consul/log/",
"log_rotate_duration":"24h",
"log_rotate_bytes":10000000,
"log_rotate_max_files":100,
"domain": "sndb",
"ports": {
"dns": 53
},
"acl": {
"enabled": true,
"default_policy": "deny",
"enable_token_persistence":true,
"tokens":{
"master":"a5f21285-42de-f589-eded-ffc5443bca00",
"default":"7a8ea281-c9a2-b559-e3df-578d46752f70"
}
},
"ui_config": {
"enabled": true
}
}
-
server 10.242.178.131 /consul/etc/server.json
{
"data_dir": "/consul/data",
"datacenter": "xingang",
"primary_datacenter":"xingang",
"log_level": "INFO",
"server": true,
"bootstrap_expect": 3,
"bind_addr": "10.242.178.131",
"client_addr": "0.0.0.0",
"recursors":["10.237.255.253"],
"start_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"retry_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"node_name": "dnsservernode2",
"encrypt": "PwGyXsuuc6w0Jx6noxi6PwrN6GeH1HHYBpZ4E77mEwk=",
"log_file":"/consul/log/",
"log_rotate_duration":"24h",
"log_rotate_bytes":10000000,
"log_rotate_max_files":100,
"domain": "sndb",
"ports": {
"dns": 53
},
"acl": {
"enabled": true,
"default_policy": "deny",
"enable_token_persistence":true,
"tokens":{
"master":"a5f21285-42de-f589-eded-ffc5443bca00",
"default":"7a8ea281-c9a2-b559-e3df-578d46752f70"
}
},
"ui_config": {
"enabled": true
}
}
-
server 10.242.178.129 /consul/etc/server.json
{
"data_dir": "/consul/data",
"datacenter": "xingang",
"primary_datacenter":"xingang",
"log_level": "INFO",
"server": true,
"bootstrap_expect": 3,
"bind_addr": "10.242.178.129",
"client_addr": "0.0.0.0",
"recursors":["10.237.255.253"],
"start_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"retry_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"node_name": "dnsservernode3",
"encrypt": "PwGyXsuuc6w0Jx6noxi6PwrN6GeH1HHYBpZ4E77mEwk=",
"log_file":"/consul/log/",
"log_rotate_duration":"24h",
"log_rotate_bytes":10000000,
"log_rotate_max_files":100,
"domain": "sndb",
"ports": {
"dns": 53
},
"acl": {
"enabled": true,
"default_policy": "deny",
"enable_token_persistence":true,
"tokens":{
"master":"a5f21285-42de-f589-eded-ffc5443bca00",
"default":"7a8ea281-c9a2-b559-e3df-578d46752f70"
}
},
"ui_config": {
"enabled": true
}
}
-
client 10.243.204.103 /opt/consul/etc/client.json
{
"data_dir": "/opt/consul/data",
"enable_script_checks": true,
"bind_addr": "10.243.204.103",
"client_addr": "0.0.0.0",
"retry_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"retry_interval": "5s",
"rejoin_after_leave": true,
"start_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"node_name": "10.243.204.103",
"datacenter": "xingang",
"primary_datacenter":"xingang",
"encrypt": "PwGyXsuuc6w0Jx6noxi6PwrN6GeH1HHYBpZ4E77mEwk=",
"log_file":"/opt/consul/log/",
"log_rotate_duration":"24h",
"log_rotate_bytes":10000000,
"log_rotate_max_files":100,
"encrypt_verify_incoming": true,
"encrypt_verify_outgoing": true,
"enable_local_script_checks":true,
"acl":{
"tokens": {
"default":"7a8ea281-c9a2-b559-e3df-578d46752f70"
}
}
}
2.5.启动集群
-
server
cd /consul nohup bin/consul agent -config-dir=etc/ &
-
client
cd /opt/consul nohup bin/consul agent -config-dir=etc/ &
2.6.状态检查
[root@lobapredb128 consul]# export CONSUL_HTTP_TOKEN=a5f21285-42de-f589-eded-ffc5443bca00 [root@lobapredb128 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode1 c30c7352-ca26-938d-9527-cb934802a94c 10.242.178.130:8300 leader true 3 dnsservernode2 0aff7e86-6ef3-63e9-6198-41046206b371 10.242.178.131:8300 follower true 3 dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 follower true 3 [root@lobapredb128 consul]# bin/consul members Node Address Status Type Build Protocol DC Segment dnsservernode1 10.242.178.130:8301 alive server 1.10.2 2 xingang <all> dnsservernode2 10.242.178.131:8301 alive server 1.10.2 2 xingang <all> dnsservernode3 10.242.178.129:8301 alive server 1.10.2 2 xingang <all> 10.243.204.103 10.243.204.103:8301 alive client 1.10.2 2 xingang <default>
2.7.域名测试
[root@lobapredb128 ~]# dig consul.service.sndb @10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.130 [root@lobapredb128 ~]# dig consul.service.sndb @10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.130 [root@lobapredb128 ~]# dig consul.service.sndb @10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.129
3.服务注册
3.1.配置文件
-
client 10.243.204.103 /opt/consul/etc/service.json
{
"services": [
{
"name": "mysqlro",
"id": "10.243.204.103_ro",
"address": "10.243.204.103",
"port": 3306,
"check": {
"id":"10.243.204.103_ro_ck",
"name":"check 3306",
"tcp": "10.243.204.103:3306",
"interval": "5s",
"timeout": "1s"
}
}
]
}
3.2.生效配置
-
client 10.243.204.103
[root@lobapredb162 ~]# /opt/consul/bin/consul reload Configuration reload triggered
3.3.域名测试
-
client 10.243.204.103
[root@lobapredb162 ~]# dig mysqlro.service.sndb @10.242.178.131 mysqlro.service.sndb. 0 IN A 10.243.204.103 [root@lobapredb162 ~]# dig mysqlro.service.sndb @10.242.178.130 mysqlro.service.sndb. 0 IN A 10.243.204.103 [root@lobapredb162 ~]# dig mysqlro.service.sndb @10.242.178.129 mysqlro.service.sndb. 0 IN A 10.243.204.103
4.集群高可用测试
4.1.leader节点宕机
-
10.242.178.130
kill server端consul进程
[root@lobapredb128 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode1 c30c7352-ca26-938d-9527-cb934802a94c 10.242.178.130:8300 leader true 3 dnsservernode2 0aff7e86-6ef3-63e9-6198-41046206b371 10.242.178.131:8300 follower true 3 dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 follower true 3 [root@lobapredb128 consul]# ps -ef|grep -i consul root 15902 1 1 11:57 ? 00:03:49 bin/consul agent -config-dir=etc/ root 20595 20016 0 16:25 pts/0 00:00:00 grep -i consul [root@lobapredb128 consul]# kill -9 15902 [root@lobapredb128 consul]# bin/consul operator raft list-peers Error getting peers: Failed to retrieve raft configuration: Get "http://127.0.0.1:8500/v1/operator/raft/configuration": dial tcp 127.0.0.1:8500: connect: connection refused
-
10.242.178.129 leader宕机后10s切换为新leader
重新选举后测试域名consul.service.sndb和mysqlro.service.sndb
[root@lobapredb127 consul]# export CONSUL_HTTP_TOKEN=a5f21285-42de-f589-eded-ffc5443bca00 [root@lobapredb127 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode2 0aff7e86-6ef3-63e9-6198-41046206b371 10.242.178.131:8300 follower true 3 dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 leader true 3 [root@lobapredb126 ~]# dig consul.service.sndb @10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.129 [root@lobapredb126 ~]# dig consul.service.sndb @10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.131 consul.service.sndb. 0 IN A 10.242.178.129 [root@lobapredb126 ~]# dig mysqlro.service.sndb @10.242.178.131 mysqlro.service.sndb. 0 IN A 10.243.204.103 [root@lobapredb126 ~]# dig mysqlro.service.sndb @10.242.178.129 mysqlro.service.sndb. 0 IN A 10.243.204.103
-
10.242.178.130
恢复原leader节点后进行域名测试
[root@lobapredb128 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode2 0aff7e86-6ef3-63e9-6198-41046206b371 10.242.178.131:8300 follower true 3 dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 leader true 3 dnsservernode1 c30c7352-ca26-938d-9527-cb934802a94c 10.242.178.130:8300 follower true 3 [root@lobapredb128 consul]# dig consul.service.sndb @10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.131 [root@lobapredb128 consul]# dig mysqlro.service.sndb @10.242.178.130 mysqlro.service.sndb. 0 IN A 10.243.204.103
4.2.follower节点宕机
-
10.242.178.131
[root@lobapredb127 consul]# ps -ef|grep -i consul root 4316 1 1 17:34 pts/0 00:00:02 bin/consul agent -config-dir=etc/ root 5148 3923 0 17:37 pts/0 00:00:00 grep -i consul [root@lobapredb127 consul]# kill -9 4316
-
10.242.178.130
[root@lobapredb128 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 follower true 3 dnsservernode1 c30c7352-ca26-938d-9527-cb934802a94c 10.242.178.130:8300 leader true 3 [root@lobapredb128 consul]# dig consul.service.sndb @10.242.178.129 consul.service.sndb. 0 IN A 10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.129 [root@lobapredb128 consul]# dig consul.service.sndb @10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.129 [root@lobapredb128 consul]# dig mysqlro.service.sndb @10.242.178.129 mysqlro.service.sndb. 0 IN A 10.243.204.103 [root@lobapredb128 consul]# dig mysqlro.service.sndb @10.242.178.130 mysqlro.service.sndb. 0 IN A 10.243.204.103
4.3.超半数节点宕机
集群当前状态
[root@lobapredb128 consul]# bin/consul operator raft list-peers Node ID Address State Voter RaftProtocol dnsservernode3 f5ff7588-6737-de5a-4f06-eb2bcbb691a4 10.242.178.129:8300 follower true 3 dnsservernode1 c30c7352-ca26-938d-9527-cb934802a94c 10.242.178.130:8300 leader true 3 dnsservernode2 0aff7e86-6ef3-63e9-6198-41046206b371 10.242.178.131:8300 follower true 3
停止两台follower节点 后集群状态,集群无法完成选举
[root@lobapredb128 consul]# bin/consul operator raft list-peers Error getting peers: Failed to retrieve raft configuration: Unexpected response code: 500 (No cluster leader) [root@lobapredb128 consul]# bin/consul members Node Address Status Type Build Protocol DC Segment dnsservernode1 10.242.178.130:8301 alive server 1.10.2 2 xingang <all> dnsservernode2 10.242.178.131:8301 left server 1.10.2 2 xingang <all> dnsservernode3 10.242.178.129:8301 failed server 1.10.2 2 xingang <all> 10.243.204.103 10.243.204.103:8301 alive client 1.10.2 2 xingang <default>
域名测试,域名解析正常，但因无leader节点，集群服务注册/探测等功能失效，生效域名为集群失效前状态
[root@lobapredb128 consul]# dig consul.service.sndb @10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.130 consul.service.sndb. 0 IN A 10.242.178.129 [root@lobapredb128 consul]# dig mysqlro.service.sndb @10.242.178.130 mysqlro.service.sndb. 0 IN A 10.243.204.103
4.4.高可用测试总结
3节点集群下高可用测试
| 测试场景 | DNS解析 | 服务注册 | 服务探测 | 说明 |
|---|---|---|---|---|
| leader节点宕机 | 正常 | 正常 | 正常 | 选举期间服务注册和探测不可用,DNS解析正常,选举时间13s |
| follower节点宕机 | 正常 | 正常 | 正常 | 单台follower节点宕机无影响 |
| 超半数节点宕机 | 正常 | 不可用 | 不可用 | 服务状态为集群状态正常情况下最终状态 |
5.访问控制
5.1.policy
初始策略,所有node和service均可向server注册
node_prefix "" {
policy="write"
}
service_prefix "" {
policy="write"
}
5.2.token
client使用基于policy生成的token与server端通信:7a8ea281-c9a2-b559-e3df-578d46752f70
{
"data_dir": "/opt/consul/data",
"enable_script_checks": true,
"bind_addr": "10.243.204.103",
"client_addr": "0.0.0.0",
"retry_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"retry_interval": "5s",
"rejoin_after_leave": true,
"start_join": ["10.242.178.129","10.242.178.130","10.242.178.131"],
"node_name": "10.243.204.103",
"datacenter": "xingang",
"primary_datacenter":"xingang",
"encrypt": "PwGyXsuuc6w0Jx6noxi6PwrN6GeH1HHYBpZ4E77mEwk=",
"log_file":"/opt/consul/log/",
"log_rotate_duration":"24h",
"log_rotate_bytes":10000000,
"log_rotate_max_files":100,
"encrypt_verify_incoming": true,
"encrypt_verify_outgoing": true,
"enable_local_script_checks":true,
"acl":{
"tokens": {
"default":"7a8ea281-c9a2-b559-e3df-578d46752f70"
}
}
}
5.3.隔离节点
-
集群状态及域名探测
[root@lobapredb128 consul]# dig mysqlro.service.sndbxingang @10.242.178.130 mysqlro.service.sndbxingang. 0 IN A 10.243.204.103 mysqlro.service.sndbxingang. 0 IN A 10.243.204.104 [root@lobapredb128 consul]# bin/consul members Node Address Status Type Build Protocol DC Segment dnsservernode1 10.242.178.130:8301 alive server 1.10.2 2 xingang <all> dnsservernode2 10.242.178.131:8301 alive server 1.10.2 2 xingang <all> dnsservernode3 10.242.178.129:8301 alive server 1.10.2 2 xingang <all> 10.243.204.103 10.243.204.103:8301 alive client 1.10.2 2 xingang <default> 10.243.204.104 10.243.204.104:8301 alive client 1.10.2 2 xingang <default>
-
更新policy
[root@lobapredb128 consul]# export CONSUL_HTTP_TOKEN=a5f21285-42de-f589-eded-ffc5443bca00
[root@lobapredb128 consul]# bin/consul acl policy read -name client -meta
ID: ab253c6d-7b8e-37f5-ac98-5be3ad7dd797
Name: client
Description:
Datacenters:
Hash: 7bcf1a65681e133eb14e11d05ec5658ffd24281f305f7bea0709dd98b5b784ab
Create Index: 11
Modify Index: 23246
Rules:
node_prefix "" { policy="write" }service_prefix "" { policy="write" }
[root@lobapredb128 consul]# bin/consul acl policy update -name client -rules 'node "10.243.204.103"{policy="deny"}node_prefix "" { policy="write" }service_prefix "" { policy="write" }'
ID: ab253c6d-7b8e-37f5-ac98-5be3ad7dd797
Name: client
Description:
Datacenters:
Rules:
node "10.243.204.103"{policy="deny"}node_prefix "" { policy="write" }service_prefix "" { policy="write" }
[root@lobapredb128 consul]# dig mysqlro.service.sndbxingang @10.242.178.130
mysqlro.service.sndbxingang. 0 IN A 10.243.204.104
5.4.解除隔离
[root@lobapredb128 consul]# bin/consul acl policy update -name client -rules 'node_prefix "" { policy="write" }service_prefix "" { policy="write" }'
ID: ab253c6d-7b8e-37f5-ac98-5be3ad7dd797
Name: client
Description:
Datacenters:
Rules:
node_prefix "" { policy="write" }service_prefix "" { policy="write" }
[root@lobapredb128 consul]# dig mysqlro.service.sndbxingang @10.242.178.130
mysqlro.service.sndbxingang. 0 IN A 10.243.204.103
mysqlro.service.sndbxingang. 0 IN A 10.243.204.104
7961
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
