实现数据分布式存储的高可用集群
redis服务器 服务器IP地址 端口
mysql50 192.168.4.50/24 无
mysql51 192.168.4.51/24 6351
mysql52 192.168.4.52/24 6352
mysql53 192.168.4.53/24 6353
mysql54 192.168.4.54/24 6354
mysql55 192.168.4.55/24 6355
mysql56 192.168.4.56/24 6356
mysql57 192.168.4.57/24 无
一、创建集群
1.6台服务器安装redis服务
[root@mysql51 ~]# yum -y install gcc
[root@mysql53 ~]# wget http://download.redis.io/releases/redis-4.0.8.tar.gz
[root@mysql51 ~]# tar -zxvf redis-4.0.8.tar.gz
[root@mysql51 ~]# cd redis-4.0.8/
[root@mysql51 redis-4.0.8]# make && make install
[root@mysql51 redis-4.0.8]# ./utils/install_server.sh
Please select the redis port for this instance: [6379] 6351
[root@mysql51 redis-4.0.8]# netstat -ntlup|grep redis-server
tcp 0 0 127.0.0.1:6351 0.0.0.0:* LISTEN 5623/redis-server 1
[root@mysql51 redis-4.0.8]# vim /etc/redis/6351.conf
70 bind 192.168.4.51
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 4096 slots | 1 slaves.
192.168.4.58:6358 (04e4a79c...) -> 1 keys | 4096 slots | 0 slaves.
[OK] 2 keys in 4 masters.
0.00 keys per slot on average.351 restart
[root@mysql51 redis-4.0.8]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> ping
PONG
2.启用每一台redis服务器的集群功能
[root@mysql51 redis-4.0.8]# vim /etc/redis/6351.conf
815 cluster-enabled yes ==>>启用居群功能
823 cluster-config-file nodes-6379.conf ==>>存储集群信息文件
829 cluster-node-timeout 5000 ==>>超时时间(单位毫秒)
[root@mysql51 redis-4.0.8]# /etc/init.d/redis_6351 restart
[root@mysql51 redis-4.0.8]# netstat -ntlup|grep redis-server
tcp 0 0 127.0.0.1:16351 0.0.0.0:* LISTEN 5710/redis-server 1 ==>>集群通信端口(=默认服务端口+10000)
tcp 0 0 127.0.0.1:6351 0.0.0.0:* LISTEN 5710/redis-server 1
[root@mysql51 redis-4.0.8]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> cluster info
[root@mysql51 redis-4.0.8]# ls /var/lib/redis/6351/
dump.rdb nodes-6379.conf
[root@mysql51 redis-4.0.8]# cat /var/lib/redis/6351/nodes-6379.conf
7ecc682d627b82bea819763e901c110bd68abb95 :0@0 myself,master - 0 0 0 connected
vars currentEpoch 0 lastVoteEpoch 0
3.配置管理主机
[root@mysql57 ~]# yum -y install rubygems ruby
[root@mysql57 ~]# gem install redis -v 3.2.1
[root@mysql57 ~]# mkdir /root/bin
[root@mysql57 ~]# tar -zxf redis-4.0.8.tar.gz
[root@mysql57 ~]# cp redis-4.0.8/src/redis-trib.rb /root/bin/
[root@mysql57 ~]# chmod +x /root/bin/redis-trib.rb
[root@mysql57 ~]# redis-trib.rb help
redis-trib.rb 命令 选项
create 创建集群
check 检查集群
info 查看集群信息
reshard 重新分片
del-node 删除主机
add-node --slave 添加slave主机
add-node 添加master主机
rebalance 平均分配hash slots
4.创建集群
--replicas 1 指定每个主服务器的从服务器个数
[root@mysql57 ~]# redis-trib.rb create --replicas 1 192.168.4.51:6351 192.168.4.52:6352 192.168.4.53:6353 192.168.4.54:6354 192.168.4.55:6355 192.168.4.56:6356
>>> Creating cluster
>>> Performing hash slots allocation on 6 nodes...
Using 3 masters:
192.168.4.51:6351
192.168.4.52:6352
192.168.4.53:6353
Adding replica 192.168.4.55:6355 to 192.168.4.51:6351
Adding replica 192.168.4.56:6356 to 192.168.4.52:6352
Adding replica 192.168.4.54:6354 to 192.168.4.53:6353
M: 7ecc682d627b82bea819763e901c110bd68abb95 192.168.4.51:6351
slots:0-5460 (5461 slots) master
M: bde8da9a3a99f3566efe3ffd7f8ff55276cb3280 192.168.4.52:6352
slots:5461-10922 (5462 slots) master
M: 40116606f9a3c7c6c5bc531f38e7911107eb4a13 192.168.4.53:6353
slots:10923-16383 (5461 slots) master
S: db992c65d0f3c6009b4b83ccabba056f188cc3ef 192.168.4.54:6354
replicates 40116606f9a3c7c6c5bc531f38e7911107eb4a13
S: 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3 192.168.4.55:6355
replicates 7ecc682d627b82bea819763e901c110bd68abb95
S: c4dd8e65b21da7b6235795951a6c66c2266ac25d 192.168.4.56:6356
replicates bde8da9a3a99f3566efe3ffd7f8ff55276cb3280
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join....
>>> Performing Cluster Check (using node 192.168.4.51:6351)
M: 7ecc682d627b82bea819763e901c110bd68abb95 192.168.4.51:6351
slots:0-5460 (5461 slots) master
1 additional replica(s)
M: bde8da9a3a99f3566efe3ffd7f8ff55276cb3280 192.168.4.52:6352
slots:5461-10922 (5462 slots) master
1 additional replica(s)
S: c4dd8e65b21da7b6235795951a6c66c2266ac25d 192.168.4.56:6356
slots: (0 slots) slave
replicates bde8da9a3a99f3566efe3ffd7f8ff55276cb3280
M: 40116606f9a3c7c6c5bc531f38e7911107eb4a13 192.168.4.53:6353
slots:10923-16383 (5461 slots) master
1 additional replica(s)
S: 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3 192.168.4.55:6355
slots: (0 slots) slave
replicates 7ecc682d627b82bea819763e901c110bd68abb95
S: db992c65d0f3c6009b4b83ccabba056f188cc3ef 192.168.4.54:6354
slots: (0 slots) slave
replicates 40116606f9a3c7c6c5bc531f38e7911107eb4a13
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
5.查看集群信息
redis-trib.rb info IP地址:端口
[root@mysql57 ~]# redis-trib.rb info 192.168.4.53:6353 ==>>查看集群信息
192.168.4.53:6353 (40116606...) -> 0 keys | 5461 slots | 1 slaves.
192.168.4.51:6351 (7ecc682d...) -> 0 keys | 5461 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 0 keys | 5462 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@mysql57 ~]# redis-trib.rb check 192.168.4.51:6351 ==>>查看集群详细信息
[root@mysql51 ~]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> cluster info ==>>查看集群信息
192.168.4.51:6351> cluster nodes ==>>查看集群详细信息
6.客户端访问集群中的主机存取数据
-c 集群模式
[root@mysql50 ~]# redis-cli -c -h 192.168.4.56 -p 6356
192.168.4.56:6356> set school tarena ==>>存数据
-> Redirected to slot [8455] located at 192.168.4.52:6352
OK
192.168.4.52:6352> exit
[root@mysql50 ~]# redis-cli -c -h 192.168.4.53 -p 6353
192.168.4.53:6353> get school ==>>取数据
-> Redirected to slot [8455] located at 192.168.4.52:6352
"tarena"
二、管理集群
1.测试故障切换
[root@mysql51 ~]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> shutdown
[root@mysql57 ~]# redis-trib.rb info 192.168.4.53:6353
192.168.4.53:6353 (40116606...) -> 0 keys | 5461 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 5462 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 1 keys | 5461 slots | 0 slaves.
[OK] 2 keys in 3 masters.
0.00 keys per slot on average.
[root@mysql51 ~]# /etc/init.d/redis_6351 start
[root@mysql57 ~]# redis-trib.rb info 192.168.4.53:6353
192.168.4.53:6353 (40116606...) -> 0 keys | 5461 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 5462 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 1 keys | 5461 slots | 1 slaves.
[OK] 2 keys in 3 masters.
0.00 keys per slot on average.
[root@mysql57 ~]# redis-trib.rb check 192.168.4.52:6352
>>> Performing Cluster Check (using node 192.168.4.52:6352)
M: bde8da9a3a99f3566efe3ffd7f8ff55276cb3280 192.168.4.52:6352
slots:5461-10922 (5462 slots) master
1 additional replica(s)
S: db992c65d0f3c6009b4b83ccabba056f188cc3ef 192.168.4.54:6354
slots: (0 slots) slave
replicates 40116606f9a3c7c6c5bc531f38e7911107eb4a13
M: 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3 192.168.4.55:6355
slots:0-5460 (5461 slots) master
1 additional replica(s)
S: c4dd8e65b21da7b6235795951a6c66c2266ac25d 192.168.4.56:6356
slots: (0 slots) slave
replicates bde8da9a3a99f3566efe3ffd7f8ff55276cb3280
M: 40116606f9a3c7c6c5bc531f38e7911107eb4a13 192.168.4.53:6353
slots:10923-16383 (5461 slots) master
1 additional replica(s)
S: 7ecc682d627b82bea819763e901c110bd68abb95 192.168.4.51:6351
slots: (0 slots) slave
replicates 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
2.添加mysql58为redis主服务器(添加的式打开集群功能的redis服务器)
redis-trib.rb add-node 添加主服务器IP地址:端口 集群中任意一个服务器IP:端口
[root@mysql57 ~]# redis-trib.rb add-node 192.168.4.58:6358 192.168.4.51:6351
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 5461 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 1 keys | 5461 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 5462 slots | 1 slaves.
192.168.4.58:6358 (04e4a79c...) -> 0 keys | 0 slots | 0 slaves.
[OK] 2 keys in 4 masters.
0.00 keys per slot on average.
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351 ==>>重新分片
How many slots do you want to move (from 1 to 16384)?4096 ==>>要移除多少个slots给新服务器
What is the receiving node ID? 04e4a79cbc4c4b269120830bf88cdeb74e333cfe ==>>新服务器的序列id
Source node #1:all ==>>从哪个服务器移除slots
Do you want to proceed with the proposed reshard plan (yes/no)? yes
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 4096 slots | 1 slaves.
192.168.4.58:6358 (04e4a79c...) -> 1 keys | 4096 slots | 0 slaves.
[OK] 2 keys in 4 masters.
0.00 keys per slot on average.
3.添加mysql59为redis从服务器(添加的式打开集群功能的redis服务器)
redis-trib.rb add-node --slave [--master-id id值] 新加服务器的IP:端口 集群中任意一个服务器IP:端口
--maseter-id 指定成为哪个服务器的从服务器,默认添加为从库最少的主库
[root@mysql57 ~]# redis-trib.rb add-node --slave 192.168.4.59:6359 192.168.4.51:6351
4.移除redis从服务器
redis-trib.rb del-node 集群中任意一个服务器IP:端口 要移除服务器的id
[root@mysql57 ~]# redis-trib.rb check 192.168.4.51:6351
S: b53b56876cde0d9b518fd4d84bc77aff61f4a3ac 192.168.4.59:6359
slots: (0 slots) slave
[root@mysql57 ~]# redis-trib.rb del-node 192.168.4.51:6351 b53b56876cde0d9b518fd4d84bc77aff61f4a3ac
>>> Removing node b53b56876cde0d9b518fd4d84bc77aff61f4a3ac from cluster 192.168.4.51:6351
>>> Sending CLUSTER FORGET messages to the cluster...
>>> SHUTDOWN the node ==>>停止了移除主机的redis服务
5.移除redis主服务器
[root@mysql57 ~]# redis-trib.rb reshard 192.168.4.51:6351
>>> Performing Cluster Check (using node 192.168.4.51:6351)
S: 7ecc682d627b82bea819763e901c110bd68abb95 192.168.4.51:6351
slots: (0 slots) slave
replicates 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3
M: 40116606f9a3c7c6c5bc531f38e7911107eb4a13 192.168.4.53:6353
slots:12288-16383 (4096 slots) master
1 additional replica(s)
S: db992c65d0f3c6009b4b83ccabba056f188cc3ef 192.168.4.54:6354
slots: (0 slots) slave
replicates 40116606f9a3c7c6c5bc531f38e7911107eb4a13
M: 5b40e764d6fe0f5be641a14f30b5b4e28d761cf3 192.168.4.55:6355
slots:1365-5460 (4096 slots) master
1 additional replica(s)
M: bde8da9a3a99f3566efe3ffd7f8ff55276cb3280 192.168.4.52:6352
slots:6827-10922 (4096 slots) master
1 additional replica(s)
M: 04e4a79cbc4c4b269120830bf88cdeb74e333cfe 192.168.4.58:6358
slots:0-1364,5461-6826,10923-12287 (4096 slots) master
0 additional replica(s)
S: c4dd8e65b21da7b6235795951a6c66c2266ac25d 192.168.4.56:6356
slots: (0 slots) slave
replicates bde8da9a3a99f3566efe3ffd7f8ff55276cb3280
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
How many slots do you want to move (from 1 to 16384)? 4096 ==>>要移除的服务器有多少个slots
What is the receiving node ID? bde8da9a3a99f3566efe3ffd7f8ff55276cb3280 ==>>将移除的slots分配给哪个服务器id
Please enter all the source node IDs.
Type 'all' to use all the nodes as source nodes for the hash slots.
Type 'done' once you entered all the source nodes IDs.
Source node #1:04e4a79cbc4c4b269120830bf88cdeb74e333cfe ==>>要移除的服务器id
Source node #2:done
Do you want to proceed with the proposed reshard plan (yes/no)? yes
[root@mysql57 ~]# redis-trib.rb del-node 192.168.4.51:6351 04e4a79cbc4c4b269120830bf88cdeb74e333cfe
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 2 keys | 8192 slots | 1 slaves.
[OK] 2 keys in 3 masters.
0.00 keys per slot on average.
4.把移除的redis服务器添加进集群
[root@mysql58 ~]# /etc/init.d/redis_6358 restart
[root@mysql58 ~]# redis-cli -h 192.168.4.58 -p 6358
192.168.4.58:6358> cluster reset ==>>重置集群状态
OK
192.168.4.58:6358> cluster info
cluster_state:fail
cluster_slots_assigned:0
cluster_slots_ok:0
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:1
cluster_size:0
cluster_current_epoch:10
cluster_my_epoch:9
cluster_stats_messages_ping_sent:41
cluster_stats_messages_sent:41
cluster_stats_messages_pong_received:41
cluster_stats_messages_received:41
然后进行添加redis主服务器操作
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 3346 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 0 keys | 3346 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 6692 slots | 1 slaves.
192.168.4.58:6358 (04e4a79c...) -> 1 keys | 3000 slots | 0 slaves.
[OK] 2 keys in 4 masters.
0.00 keys per slot on average.
[root@mysql57 ~]# redis-trib.rb rebalance 192.168.4.51:6351 ==>>将占有slots的主服务器进行slots的平均分配
[root@mysql57 ~]# redis-trib.rb info 192.168.4.51:6351
192.168.4.53:6353 (40116606...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.55:6355 (5b40e764...) -> 0 keys | 4096 slots | 1 slaves.
192.168.4.52:6352 (bde8da9a...) -> 1 keys | 4096 slots | 1 slaves.
192.168.4.58:6358 (04e4a79c...) -> 1 keys | 4096 slots | 0 slaves.
[OK] 2 keys in 4 masters.
0.00 keys per slot on average.
5.把集群中的主机还原成独立的服务器
[root@mysql51 ~]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> shutdown
not connected> exit
[root@mysql51 ~]# /etc/init.d/redis_6351 stop
/var/run/redis_6351.pid does not exist, process is not running
[root@mysql51 ~]# rm -fr /var/lib/redis/6351/*
[root@mysql51 ~]# vim /etc/redis/6351.conf
815 # cluster-enabled yes
823 #cluster-config-file nodes-6379.conf
829 # cluster-node-timeout 5000
[root@mysql51 ~]# /etc/init.d/redis_6351 start
Starting Redis server...
[root@mysql51 ~]# netstat -ntlup|grep redis
tcp 0 0 192.168.4.51:6351 0.0.0.0:* LISTEN 22204/redis-server
[root@mysql51 ~]# redis-cli -h 192.168.4.51 -p 6351
192.168.4.51:6351> cluster info
ERR This instance has cluster support disabled