redis：sentinel配置及测试

参考原理：

http://www.redis.cn/topics/sentinel.html
https://segmentfault.com/a/1190000002680804

结构：

192.168.56.159 6379 master
192.168.56.159 6380 slave
192.168.56.159 26379 sentinel
192.168.56.159 26380 sentinel

简单测试

sentinel配置文件

port 26379
daemonize yes
logfile "/data/redis/redis_sentinel/26379/sentinel.log"
dir "/data/redis/redis_sentinel/26379"

sentinel monitor mymaster 192.168.56.159 6379 1
sentinel down-after-milliseconds mymaster 15000
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 180000
sentinel auth-pass mymaster xxx
sentinel client-reconfig-script mymaster /data/scripts/redis.sh

启动sentinel
redis-sentinel sentinel.conf
查看日志
6622:X 25 Jul 15:14:20.113 # Sentinel runid is 6b52acf5d95573a1f0b629d5aa2e203ca5be8fa9
6622:X 25 Jul 15:14:20.113 # +monitor master mymaster 192.168.56.159 6379 quorum 1
6622:X 25 Jul 15:14:20.114 * +slave slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379

再启动26380端口的sentinel，查看26379的日志
6622:X 25 Jul 15:18:33.936 * +sentinel sentinel 192.168.56.159:26380 192.168.56.159 26380 @ mymaster 192.168.56.159 6379

登录任意端口的sentinel查看信息
[root@192.168.56.159 scripts]redis-cli -p 26380 info Sentinel
# Sentinel
sentinel_masters:1
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
master0:name=mymaster,status=ok,address=192.168.56.159:6379,slaves=1,sentinels=2

测试：

停掉slave 6380
6622:X 25 Jul 15:21:35.262 # +sdown slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379

启动slave 6380
6622:X 25 Jul 15:22:36.074 * +reboot slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6622:X 25 Jul 15:22:36.130 # -sdown slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379

停掉sentinel 26379
6666:X 25 Jul 15:24:03.631 # +sdown sentinel 192.168.56.159:26379 192.168.56.159 26379 @ mymaster 192.168.56.159 6379
启动sentinel 26379
6666:X 25 Jul 15:24:35.313 # -sdown sentinel 192.168.56.159:26379 192.168.56.159 26379 @ mymaster 192.168.56.159 6379
6666:X 25 Jul 15:24:37.315 * -dup-sentinel master mymaster 192.168.56.159 6379 #duplicate of 192.168.56.159:26379 or e64490cfd0f4e1cb2c3a7be53657a00c68defe99
6666:X 25 Jul 15:24:37.315 * +sentinel sentinel 192.168.56.159:26379 192.168.56.159 26379 @ mymaster 192.168.56.159 6379

停掉master 6379
sentinel-26379日志
6732:X 25 Jul 15:28:21.904 # +sdown master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:21.904 # +odown master mymaster 192.168.56.159 6379 #quorum 1/1
6732:X 25 Jul 15:28:21.904 # +new-epoch 1
6732:X 25 Jul 15:28:21.905 # +try-failover master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:21.916 # +vote-for-leader e64490cfd0f4e1cb2c3a7be53657a00c68defe99 1
6732:X 25 Jul 15:28:21.928 # 192.168.56.159:26380 voted for e64490cfd0f4e1cb2c3a7be53657a00c68defe99 1
6732:X 25 Jul 15:28:21.993 # +elected-leader master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:21.993 # +failover-state-select-slave master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:22.085 # +selected-slave slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:22.085 * +failover-state-send-slaveof-noone slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:22.161 * +failover-state-wait-promotion slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:22.982 # +promoted-slave slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:22.982 # +failover-state-reconf-slaves master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:23.008 # +failover-end master mymaster 192.168.56.159 6379
6732:X 25 Jul 15:28:23.008 # +switch-master mymaster 192.168.56.159 6379 192.168.56.159 6380
6732:X 25 Jul 15:28:23.008 * +slave slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 15:28:38.039 # +sdown slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
sentinel-26380日志
6666:X 25 Jul 15:28:21.924 # +new-epoch 1
6666:X 25 Jul 15:28:21.927 # +vote-for-leader e64490cfd0f4e1cb2c3a7be53657a00c68defe99 1
6666:X 25 Jul 15:28:21.933 # +sdown master mymaster 192.168.56.159 6379
6666:X 25 Jul 15:28:21.933 # +odown master mymaster 192.168.56.159 6379 #quorum 1/1
6666:X 25 Jul 15:28:21.934 # Next failover delay: I will not start a failover before Mon Jul 25 15:34:22 2016
6666:X 25 Jul 15:28:23.019 # +config-update-from sentinel 192.168.56.159:26379 192.168.56.159 26379 @ mymaster 192.168.56.159 6379
6666:X 25 Jul 15:28:23.019 # +switch-master mymaster 192.168.56.159 6379 192.168.56.159 6380
6666:X 25 Jul 15:28:23.019 * +slave slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6666:X 25 Jul 15:28:38.099 # +sdown slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380

slave 6380状态改变
原状态
# Replication
role:slave
master_host:192.168.56.159
master_port:6379
master_link_status:up
现状态
[root@192.168.56.159 redis6380]redis-cli -p 6380 -a fangyali info Replication
# Replication
role:master
connected_slaves:0
master_repl_offset:0


在启动原master 6379
6732:X 25 Jul 15:34:23.247 # -sdown slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 15:34:33.185 * +convert-to-slave slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
查看信息(原master配置文件masterauth 没有打开，第一次切换master_link_status:down)
redis-cli -p 6379 -a fangyali info Replication  
# Replication
role:slave
master_host:192.168.56.159
master_port:6380
master_link_status:up

故障转移

一次故障转移操作由以下步骤组成：
发现主服务器已经进入客观下线状态。
对我们的当前纪元进行自增（详情请参考 Raft leader election ）， 并尝试在这个纪元中当选。
如果当选失败， 那么在设定的故障迁移超时时间的两倍之后， 重新尝试当选。 如果当选成功， 那么执行以下步骤。
选出一个从服务器，并将它升级为主服务器。
向被选中的从服务器发送 SLAVEOF NO ONE 命令，让它转变为主服务器。
通过发布与订阅功能， 将更新后的配置传播给所有其他 Sentinel ， 其他 Sentinel 对它们自己的配置进行更新。
向已下线主服务器的从服务器发送 SLAVEOF 命令， 让它们去复制新的主服务器。
当所有从服务器都已经开始复制新的主服务器时， 领头 Sentinel 终止这次故障迁移操作。

故障转移(failover)的第一步，就是选出新的master，大致的筛选流程为：
1. 删除列表中所有处于下线或者断线状态的slave
2. 删除列表中所有最近五秒内没有回复过领头sentinel的INFO命令的slave
3. 删除所有与已下线主服务器连接断开超过down-after-milliseconds * 10毫秒的slave（确保slave没有过早与master断开，副本比较新）
4. 根据slave优先级选择
5. 如果优先级相同，选择复制偏移量最大的slave
6. 如果都相同，按照run_id排序，选出run_id最小的slave

有一次failover会失败，然后等待2*failover-timeout 后failover成功，原因？

sentinel-26379日志
6732:X 25 Jul 18:40:37.402 # +sdown master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:40:37.402 # +odown master mymaster 192.168.56.159 6380 #quorum 1/1
6732:X 25 Jul 18:40:37.403 # +new-epoch 2
6732:X 25 Jul 18:40:37.403 # +try-failover master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:40:37.455 # +vote-for-leader e64490cfd0f4e1cb2c3a7be53657a00c68defe99 2
6732:X 25 Jul 18:40:37.456 # 192.168.56.159:26380 voted for de9a51fdc0ae15c8bc09cbde6036209d63c44988 2
6732:X 25 Jul 18:40:48.162 # -failover-abort-not-elected master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:40:48.239 # Next failover delay: I will not start a failover before Mon Jul 25 18:46:38 2016
6732:X 25 Jul 18:46:38.184 # +new-epoch 3
6732:X 25 Jul 18:46:38.184 # +try-failover master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.188 # +vote-for-leader e64490cfd0f4e1cb2c3a7be53657a00c68defe99 3
6732:X 25 Jul 18:46:38.223 # 192.168.56.159:26380 voted for e64490cfd0f4e1cb2c3a7be53657a00c68defe99 3
6732:X 25 Jul 18:46:38.252 # +elected-leader master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.252 # +failover-state-select-slave master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.324 # +selected-slave slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.324 * +failover-state-send-slaveof-noone slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.400 * +failover-state-wait-promotion slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.559 # +promoted-slave slave 192.168.56.159:6379 192.168.56.159 6379 @ mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.560 # +failover-state-reconf-slaves master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.647 # +failover-end master mymaster 192.168.56.159 6380
6732:X 25 Jul 18:46:38.647 # +switch-master mymaster 192.168.56.159 6380 192.168.56.159 6379
6732:X 25 Jul 18:46:38.647 * +slave slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6732:X 25 Jul 18:46:53.670 # +sdown slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
sentinel-26380日志
6666:X 25 Jul 18:40:37.413 # +sdown master mymaster 192.168.56.159 6380
6666:X 25 Jul 18:40:37.413 # +odown master mymaster 192.168.56.159 6380 #quorum 1/1
6666:X 25 Jul 18:40:37.413 # +new-epoch 2
6666:X 25 Jul 18:40:37.413 # +try-failover master mymaster 192.168.56.159 6380
6666:X 25 Jul 18:40:37.454 # +vote-for-leader de9a51fdc0ae15c8bc09cbde6036209d63c44988 2
6666:X 25 Jul 18:40:37.456 # 192.168.56.159:26379 voted for e64490cfd0f4e1cb2c3a7be53657a00c68defe99 2
6666:X 25 Jul 18:40:48.312 # -failover-abort-not-elected master mymaster 192.168.56.159 6380
6666:X 25 Jul 18:40:48.403 # Next failover delay: I will not start a failover before Mon Jul 25 18:46:38 2016
6666:X 25 Jul 18:46:38.195 # +new-epoch 3
6666:X 25 Jul 18:46:38.221 # +vote-for-leader e64490cfd0f4e1cb2c3a7be53657a00c68defe99 3
6666:X 25 Jul 18:46:38.222 # Next failover delay: I will not start a failover before Mon Jul 25 18:52:38 2016
6666:X 25 Jul 18:46:38.654 # +config-update-from sentinel 192.168.56.159:26379 192.168.56.159 26379 @ mymaster 192.168.56.159 6380
6666:X 25 Jul 18:46:38.654 # +switch-master mymaster 192.168.56.159 6380 192.168.56.159 6379
6666:X 25 Jul 18:46:38.658 * +slave slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379
6666:X 25 Jul 18:46:53.674 # +sdown slave 192.168.56.159:6380 192.168.56.159 6380 @ mymaster 192.168.56.159 6379