redis cluster 集群配置

环境

三主三从共6个节点（这里用三台虚拟机实现，用一台机器不同的端口6001、6002分别做不同的节点）

修改主机名（所有节点均做）

hostnamectl set-hostname redis1
hostnamectl set-hostname redis2
hostnamectl set-hostname redis3

地址解析（所有节点均做）

cat  >>  /etc/hosts  << EOF
192.168.122.37 redis1
192.168.122.175 redis2
192.168.122.174 redis3
EOF

scp /etc/hosts redis2:/etc/hosts
scp /etc/hosts redis3:/etc/hosts

修改系统参数（所有节点均做）

1、修改最大可打开文件数

cat >> /etc/security/limits.conf << EOF
* soft nofile 102400
* hard nofile 102400
EOF

scp /etc/security/limits.conf redis2:/etc/security/limits.conf
scp /etc/security/limits.conf redis3:/etc/security/limits.conf

2、TCP监听队列大小

echo "net.core.somaxconn = 32767" >> /etc/sysctl.conf

#让上一条命令参数永久生效
sysctl -p
#立即生效
sysctl -w 

3、OOM相关：vm.overcommit_memory

echo “vm.overcommit_memory=1” >> /etc/sysctl.conf
sysctl -p

4、开启内核的“Transparent Huge Pages (THP)”特性

#默认是always，修改为never
echo never > /sys/kernel/mm/transparent_hugepage/enabled

#设置永久生效将下面命令加入/etc/rc.local
#fstab是在系统启动之前进行挂载，rc.local是在系统启动之后
echo “echo never > /sys/kernel/mm/transparent_hugepage/enabled” >> /etc/rc.local
[root@redis01 ~]# chmod +x /etc/rc.local

安装redis（编译）

   28  wget http://download.redis.io/releases/redis-6.0.5.tar.gz
   29  tar -zxvf redis-6.0.5.tar.gz  -C /usr/local
   30  cd redis-6.0.5/
   31  make
   32  make install PREFIX=/usr/local/redis-cluster  #指定安装目录
   33  mkdir -p /redis/{6001,6002}/{conf,data,log}
  
  #编辑6001（redis1-master）配置文件 
   34  cd /redis/6001/conf
   35  vim redis.conf
bind 0.0.0.0
protected-mode no
port 6001
dir /redis/6001/data
cluster-enabled yes
cluster-config-file /redis/6001/conf/nodes.conf
cluster-node-timeout 5000
appendonly yes
daemonize yes
#requirepass redis（把这个密码去掉，不然后面会拒绝连接）
pidfile /redis/6001/redis.pid
logfile /redis/6001/log/redis.log
EOF

 #编辑6002（redis1-slave）配置文件    
   36  sed 's/6001/6002/g' redis.conf > /redis/6002/conf/redis.conf

#启动脚本 start-redis-cluster.sh
   37  cat >/usr/local/redis-cluster/start-redis-cluster.sh<<-EOFREDIS_HOME=/usr/local/redis-cluster
REDIS_CONF=/redis
\$REDIS_HOME/bin/redis-server \$REDIS_CONF/6001/conf/redis.conf
\$REDIS_HOME/bin/redis-server \$REDIS_CONF/6002/conf/redis.conf
EOF

#添加权限
   38  chmod +x /usr/local/redis-cluster/start-redis-cluster.sh
#启动redis
   39  bash /usr/local/redis-cluster/start-redis-cluster.sh
#查看端口
   40  ss -anput | grep redis
LISTEN     0      511          *:6001                     *:*                   users:(("redis-server",pid=15080,fd=6))
LISTEN     0      511          *:6002                     *:*                   users:(("redis-server",pid=15085,fd=6))
LISTEN     0      511          *:16001                    *:*                   users:(("redis-server",pid=15080,fd=9))
LISTEN     0      511          *:16002                    *:*                   users:(("redis-server",pid=15085,fd=9))
[root@redis2-175 bin]#  ps -ef|grep redis

创建 redis cluster

   cd /usr/local/redis-cluster/bin
   ./redis-cli --cluster create 192.168.122.37:6001 192.168.122.37:6002 192.168.122.175:6001 192.168.122.175:6002 192.168.122.174:6001 192.168.122.174:6002 --cluster-replicas 1
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 192.168.122.175:6002 to 192.168.122.37:6001
Adding replica 192.168.122.174:6002 to 192.168.122.175:6001
Adding replica 192.168.122.37:6002 to 192.168.122.174:6001
M: 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a 192.168.122.37:6001
   slots:[0-5460] (5461 slots) master
S: 1b69d78400b94b582cd6f58076cb55ca4de57039 192.168.122.37:6002
   replicates 0691b794652ca266c1cc3a7cb724a5cbc69a4669
M: 4ab89148c62a581a1010acef3ea04f362be6d33f 192.168.122.175:6001
   slots:[5461-10922] (5462 slots) master
S: 96b983025797167e47cc94116d8c766eafb08970 192.168.122.175:6002
   replicates 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a
M: 0691b794652ca266c1cc3a7cb724a5cbc69a4669 192.168.122.174:6001
   slots:[10923-16383] (5461 slots) master
S: 65febcf348984eac47560e97d50065ebd7d3d4ee 192.168.122.174:6002
   replicates 4ab89148c62a581a1010acef3ea04f362be6d33f
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.
>>> Performing Cluster Check (using node 192.168.122.37:6001)
M: 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a 192.168.122.37:6001
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: 96b983025797167e47cc94116d8c766eafb08970 192.168.122.175:6002
   slots: (0 slots) slave
   replicates 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a
M: 0691b794652ca266c1cc3a7cb724a5cbc69a4669 192.168.122.174:6001
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 1b69d78400b94b582cd6f58076cb55ca4de57039 192.168.122.37:6002
   slots: (0 slots) slave
   replicates 0691b794652ca266c1cc3a7cb724a5cbc69a4669
M: 4ab89148c62a581a1010acef3ea04f362be6d33f 192.168.122.175:6001
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 65febcf348984eac47560e97d50065ebd7d3d4ee 192.168.122.174:6002
   slots: (0 slots) slave
   replicates 4ab89148c62a581a1010acef3ea04f362be6d33f
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

如果这一步发生错误，停掉服务（kill -9），删除其产生的相关数据（所有节点）
rm -rf /redis/6001/data/* && rm -rf /redis/6001/log/* && rm -rf /redis/6002/log/* && rm -rf /redis/6002/data/* && rm -rf /redis/6002/confdes.conf && rm -rf /redis/6001/confdes.conf

软链接

ln -s /usr/local/redis-cluster/bin/redis-cli /bin/redis-cli

查看集群中的节点

 redis-cli -c -p 6001
127.0.0.1:6001> cluster nodes
96b983025797167e47cc94116d8c766eafb08970 192.168.122.175:6002@16002 slave 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a 0 1600259279474 4 connected
0691b794652ca266c1cc3a7cb724a5cbc69a4669 192.168.122.174:6001@16001 master - 0 1600259280000 5 connected 10923-16383
1b69d78400b94b582cd6f58076cb55ca4de57039 192.168.122.37:6002@16002 slave 0691b794652ca266c1cc3a7cb724a5cbc69a4669 0 1600259279975 5 connected
4ab89148c62a581a1010acef3ea04f362be6d33f 192.168.122.175:6001@16001 master - 0 1600259279000 3 connected 5461-10922
65febcf348984eac47560e97d50065ebd7d3d4ee 192.168.122.174:6002@16002 slave 4ab89148c62a581a1010acef3ea04f362be6d33f 0 1600259280480 6 connected
823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a 192.168.122.37:6001@16001 myself,master - 0 1600259278000 1 connected 0-5460

验证集群

在redis1上验证

127.0.0.1:6001> set name aaa
-> Redirected to slot [5798] located at 192.168.122.175:6001
OK
192.168.122.175:6001> get name
"aaa"
192.168.122.175:6001> exit
[root@redis1-37 bin]#  /usr/local/redis-cluster/bin/redis-cli -c -p 6002
127.0.0.1:6002> get name
-> Redirected to slot [5798] located at 192.168.122.175:6001
"aaa"
192.168.122.175:6001> get name
"aaa"
192.168.122.175:6001> exit
[root@redis1-37 bin]#  /usr/local/redis-cluster/bin/redis-cli -c -p 6002
127.0.0.1:6002> get name
-> Redirected to slot [5798] located at 192.168.122.175:6001
"aaa"
192.168.122.175:6001> 
192.168.122.175:6001> exit

在redis2上验证

[root@redis2-175 bin]# redis-cli -c -p 6001
127.0.0.1:6001> get name3 ccc
(error) ERR wrong number of arguments for 'get' command
127.0.0.1:6001> set name3 ccc
-> Redirected to slot [4807] located at 192.168.122.37:6001
OK
192.168.122.37:6001> get name
-> Redirected to slot [5798] located at 192.168.122.175:6001
"aaa"
192.168.122.175:6001> exit
[root@redis2-175 bin]# redis-cli -c -p 6002
127.0.0.1:6002> get name
-> Redirected to slot [5798] located at 192.168.122.175:6001
"aaa"
192.168.122.175:6001> get name3
-> Redirected to slot [4807] located at 192.168.122.37:6001
"ccc"
192.168.122.37:6001> exit
[root@redis2-175 bin]# redis-cli -c -p 6002
127.0.0.1:6002> set bbb 222
-> Redirected to slot [5287] located at 192.168.122.37:6001
OK
192.168.122.37:6001> get bbb
"222"
192.168.122.37:6001> exit

检查节点状态

[root@redis3-174 bin]# redis-cli --cluster check 192.168.122.174:6001
192.168.122.174:6001 (0691b794...) -> 0 keys | 5461 slots | 1 slaves.
192.168.122.175:6001 (4ab89148...) -> 1 keys | 5462 slots | 1 slaves.
192.168.122.37:6001 (823fb08f...) -> 2 keys | 5461 slots | 1 slaves.
[OK] 3 keys in 3 masters.
0.00 keys per slot on average.
>>> Performing Cluster Check (using node 192.168.122.174:6001)
M: 0691b794652ca266c1cc3a7cb724a5cbc69a4669 192.168.122.174:6001
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 96b983025797167e47cc94116d8c766eafb08970 192.168.122.175:6002
   slots: (0 slots) slave
   replicates 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a
S: 65febcf348984eac47560e97d50065ebd7d3d4ee 192.168.122.174:6002
   slots: (0 slots) slave
   replicates 4ab89148c62a581a1010acef3ea04f362be6d33f
S: 1b69d78400b94b582cd6f58076cb55ca4de57039 192.168.122.37:6002
   slots: (0 slots) slave
   replicates 0691b794652ca266c1cc3a7cb724a5cbc69a4669
M: 4ab89148c62a581a1010acef3ea04f362be6d33f 192.168.122.175:6001
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
M: 823fb08fc6a4c9fb16f9caa3845ad0ba1737a12a 192.168.122.37:6001
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
[root@redis3-174 bin]#