集群:
是一个提供多个Redis(分布式)节点间共享数据的程序集。
集群部署
Redis 集群的键空间被分割为 16384 hash个槽(slot), 集群的最大节点数量也是 16384 个
一、单机集群:
哈希槽 < 16834 : 集群损坏
master down过半 : 集群损坏
1、配置单机多线程,以7001为例
[root@server2 ~]# mkdir /usr/local/redis-cluster/700{1..6} -p
[root@server2 ~]# cd /usr/local/redis-cluster/
[root@server2 redis-cluster]# ls
7001 7002 7003 7004 7005 7006
[root@server2 7001]# vim redis.conf
port 7001
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
appendonly yes
dir /usr/local/redis-cluster/7001
logfile /usr/local/redis-cluster/7001/redis.log
daemonize yes
pidfile /usr/local/redis-cluster/redis.pid
## 启动redis
[root@server2 7001]# redis-server redis.conf
7002/7003/7004/7005/7006依次进行配置
[root@server2 7001]# cp redis.conf ../7002/
[root@server2 7001]# cp redis.conf ../7003/
[root@server2 7001]# cp redis.conf ../7004/
[root@server2 7001]# cp redis.conf ../7005/
[root@server2 7001]# cp redis.conf ../7006/
完成之后查看进程: ps ax
1452 ? Ssl 0:00 redis-server *:7001 [cluster]
1459 ? Ssl 0:00 redis-server *:7002 [cluster]
1465 ? Ssl 0:00 redis-server *:7003 [cluster]
1471 ? Ssl 0:00 redis-server *:7004 [cluster]
1477 ? Ssl 0:00 redis-server *:7005 [cluster]
1483 ? Ssl 0:00 redis-server *:7006 [cluster]
2、配置 redis-trib.rb (需要ruby支持)
设置 redis-trib.rb 命令
[root@server2 redis-cluster]# cd ~/redis/redis-4.0.1/src/
[root@server2 src]# file redis-trib.rb
redis-trib.rb: a /usr/bin/env ruby script text executable
[root@server2 src]# cp redis-trib.rb /usr/local/bin/
安装ruby,注意:当前适用版本为 2.2
如果yum安装了低版本,update时,需要先安装libyaml
yum install -y ruby-2.2.3-1.el6.x86_64.rpm libyaml-0.1.3-4.el6_6.x86_64.rpm
导入redis模块,注意:需要 redis-4.0.1.gem
[root@server2 redis]# gem list | grep redis
[root@server2 redis]# gem install --local redis-4.0.1.gem
Successfully installed redis-4.0.1
Parsing documentation for redis-4.0.1
Installing ri documentation for redis-4.0.1
Done installing documentation for redis after 1 seconds
1 gem installed
[root@server2 redis]# gem list | grep redis
redis (4.0.1)
3、集群化配置
[root@server2 redis]# redis-trib.rb create --replicas 1 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005 127.0.0.1:7006
>>> Creating cluster
>>> Performing hash slots allocation on 6 nodes...
Using 3 masters:
127.0.0.1:7001
127.0.0.1:7002
127.0.0.1:7003
Adding replica 127.0.0.1:7004 to 127.0.0.1:7001
Adding replica 127.0.0.1:7005 to 127.0.0.1:7002
Adding replica 127.0.0.1:7006 to 127.0.0.1:7003
>>> Performing Cluster Check (using node 127.0.0.1:7001)
M: 143e9c5e082090c5ecd7ec70ce66b35b451fddcf 127.0.0.1:7001
slots:0-5460 (5461 slots) master
1 additional replica(s)
M: c257a17cbd0cdec02f28793d12df6c35c1ad0105 127.0.0.1:7003
slots:10923-16383 (5461 slots) master
1 additional replica(s)
S: aa3f22248112829028a32ed5c458dd9ea062f8a8 127.0.0.1:7006
slots: (0 slots) slave
replicates c257a17cbd0cdec02f28793d12df6c35c1ad0105
S: 55f3959fa8f06108ef439b017eda56e61a1089d5 127.0.0.1:7005
slots: (0 slots) slave
replicates ce68233c661aa1b8fd275853ca987ec520c5cb38
M: ce68233c661aa1b8fd275853ca987ec520c5cb38 127.0.0.1:7002
slots:5461-10922 (5462 slots) master
1 additional replica(s)
S: c8bfac6864ce47afdc9d36268dc548aacd5972b6 127.0.0.1:7004
slots: (0 slots) slave
replicates 143e9c5e082090c5ecd7ec70ce66b35b451fddcf
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
4、测试
任何节点操作,都会有 redis 集群分配,自动调转
[root@server2 redis]# redis-cli -c -p 7001
127.0.0.1:7001> set name westos
-> Redirected to slot [5798] located at 127.0.0.1:7002
OK
127.0.0.1:7002> get name
"westos"
127.0.0.1:7002> exit
[root@server2 redis]# redis-cli -c -p 7004
127.0.0.1:7004> get name
-> Redirected to slot [5798] located at 127.0.0.1:7002
"westos"
127.0.0.1:7002> set name redhat
OK
127.0.0.1:7002> exit
[root@server2 redis]# redis-cli -c -p 7006
127.0.0.1:7006> get name
-> Redirected to slot [5798] located at 127.0.0.1:7002
"redhat"
127.0.0.1:7002>
5、损坏测试
破坏 1 个 master,slave 自动接管
[root@server2 redis]# ps ax | grep 700
1073 ? Ssl 0:02 redis-server *:7001 [cluster]
1085 ? Ssl 0:02 redis-server *:7002 [cluster]
1092 ? Ssl 0:02 redis-server *:7003 [cluster]
1098 ? Ssl 0:02 redis-server *:7004 [cluster]
1104 ? Ssl 0:02 redis-server *:7005 [cluster]
1110 ? Ssl 0:02 redis-server *:7006 [cluster]
1155 pts/0 S+ 0:00 grep 700
[root@server2 redis]# kill 1073 ##master 7001 down
[root@server2 redis]# redis-trib.rb info 127.0.0.1:7006
127.0.0.1:7002 (e38f5293...) -> 1 keys | 5462 slots | 1 slaves.
127.0.0.1:7003 (e29b054a...) -> 0 keys | 5461 slots | 1 slaves.
127.0.0.1:7004 (85742eee...) -> 0 keys | 5461 slots | 0 slaves. ##slave 7004 接管 master
[OK] 1 keys in 3 masters.
0.00 keys per slot on average.
[root@server2 redis]# redis-cli -c -p 7006
127.0.0.1:7006> get name ##集群正常
-> Redirected to slot [5798] located at 127.0.0.1:7002
"redhat"
破坏 1 组 master,slave (1组master、slave down后,集群无法使用)
[root@server2 redis]# ps ax | grep 700
1085 ? Ssl 0:02 redis-server *:7002 [cluster]
1092 ? Ssl 0:02 redis-server *:7003 [cluster]
1098 ? Ssl 0:02 redis-server *:7004 [cluster]
1104 ? Ssl 0:02 redis-server *:7005 [cluster]
1110 ? Ssl 0:02 redis-server *:7006 [cluster]
1160 pts/0 S+ 0:00 grep 700
[root@server2 redis]# kill 1098
##破坏一组master-slave之后,哈希槽不够,集群损坏
[root@server2 redis]# redis-cli -c -p 7006
127.0.0.1:7006> get name
(error) CLUSTERDOWN The cluster is down
127.0.0.1:7006> exit
[root@server2 redis]# redis-trib.rb check 127.0.0.1:7005
>>> Performing Cluster Check (using node 127.0.0.1:7005)
S: 11968d3c17416bca93dcb902012f092aabd82c34 127.0.0.1:7005
slots: (0 slots) slave
replicates e38f5293ee0598cab9772bf1c1d3a2bbf6688902
M: e38f5293ee0598cab9772bf1c1d3a2bbf6688902 127.0.0.1:7002
slots:5461-10922 (5462 slots) master
1 additional replica(s)
M: e29b054ae705e7956fe660022f2df5ce7aa6e40b 127.0.0.1:7003
slots:10923-16383 (5461 slots) master
1 additional replica(s)
S: d7a3c0dc89de44fcd0d826657e38acae5e64b8c4 127.0.0.1:7006
slots: (0 slots) slave
replicates e29b054ae705e7956fe660022f2df5ce7aa6e40b
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[ERR] Not all 16384 slots are covered by nodes.
破坏过半的master,集群也会损坏
二、Mysql+redis
Server2 配置php+nginx
Server4 配置redis
Server3 配置mysql(用5.7版本以下的)
1、Worker(server2)
##安装nginx、php-fpm
[root@server2 redis]# yum install -y php-5.3.3-38.el6.x86_64.rpm php-cli-5.3.3-38.el6.x86_64.rpm php-common-5.3.3-38.el6.x86_64.rpm php-devel-5.3.3-38.el6.x86_64.rpm
php-fpm-5.3.3-38.el6.x86_64.rpm php-gd-5.3.3-38.el6.x86_64.rpm php-mbstring-5.3.3-38.el6.x86_64.rpm php-mysql-5.3.3-38.el6.x86_64.rpm php-pdo-5.3.3-38.el6.x86_64.rpm
[root@server2 redis]# rpm -ivh nginx-1.8.0-1.el6.ngx.x86_64.rpm
warning: nginx-1.8.0-1.el6.ngx.x86_64.rpm: Header V4 RSA/SHA1 Signature, key ID 7bd9bf62: NOKEY
Preparing... ########################################### [100%]
1:nginx ########################################### [100%]
[root@server2 redis]# vim /etc/php.ini
946 date.timezone = Asia/Shanghai
[root@server2 redis]# vim /etc/php-fpm.d/www.conf
39 user = nginx
41 group = nginx
[root@server2 redis]# id nginx
uid=498(nginx) gid=499(nginx) groups=499(nginx)
[root@server2 ~]# /etc/init.d/nginx start
Starting nginx: [ OK ]
[root@server2 ~]# /etc/init.d/php-fpm start
Starting php-fpm: [ OK ]
[root@server2 ~]# netstat -antpl | grep nginx
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 1300/nginx
[root@server2 ~]# netstat -antpl | grep php
tcp 0 0 127.0.0.1:9000 0.0.0.0:* LISTEN 1309/php-fpm
##配置nginx、php-fpm
[root@server2 ~]# vim /etc/nginx/conf.d/default.conf
10 index index.php index.html index.htm;
30 location ~ \.php$ {
31 root html;
32 fastcgi_pass 127.0.0.1:9000;
33 fastcgi_index index.php;
34 fastcgi_param SCRIPT_FILENAME /usr/share/nginx/html/$fastcgi_script_name;
35 include fastcgi_params;
36 }
[root@server2 ~]# cd /usr/share/nginx/html/
[root@server2 html]# vim index.php
<?php
phpinfo()
?>
[root@server2 html]# /etc/init.d/nginx reload
Reloading nginx: [ OK ]
##配置redis
[root@server2 html]# cp ~/redis/test.php .
[root@server2 html]# ls
50x.html index.html index.php test.php
[root@server2 html]# vim test.php
3 $redis->connect('172.25.120.4',6379) or die ("could net connect redis server");
10 $connect = mysql_connect('172.25.120.3','redis','wes tos');
[root@server2 redis]# unzip phpredis-master.zip
[root@server2 redis]# cd phpredis-master
[root@server2 phpredis-master]# phpize
Configuring for:
PHP Api Version: 20090626
Zend Module Api No: 20090626
Zend Extension Api No: 220090626
[root@server2 phpredis-master]# ./configure
[root@server2 phpredis-master]# make && make install
[root@server2 phpredis-master]# cd /usr/lib64/php/modules/
[root@server2 modules]# ls
curl.so json.so mysql.so pdo_sqlite.so sqlite3.so
fileinfo.so mbstring.so pdo_mysql.so phar.so zip.so
gd.so mysqli.so pdo.so redis.so
[root@server2 modules]# cd /etc/php.d/
[root@server2 php.d]# cp mysql.ini redis.ini
[root@server2 php.d]# vim redis.ini
[root@server2 php.d]# cat redis.ini
extension=redis.so
[root@server2 php.d]# /etc/init.d/php-fpm reload
Reloading php-fpm: [12-Jul-2018 13:56:53] NOTICE: configuration file /etc/php-fpm.conf test is successful
[ OK ]
[root@server2 php.d]# php -m | grep redis
redis
nginx 配置ok
访问:http://172.25.32.2/index.php
2、redis-master
Redis(server4)
server4为master,server3为slave即可
3、Mysql(server3)
[root@server4 ~]# yum install -y mysql-server
[root@server4 ~]# /etc/init.d/mysqld start
[root@server3 ~]# cd redis/
[root@server3 redis]# vim test.sql
[root@server3 redis]# mysql < test.sql
[root@server3 redis]# mysql
mysql> use test;
mysql> select * from test;
+----+-------+
| id | name |
+----+-------+
| 1 | test1 |
| 2 | test2 |
| 3 | test3 |
| 4 | test4 |
| 5 | test5 |
| 6 | test6 |
| 7 | test7 |
| 8 | test8 |
| 9 | test9 |
+----+-------+
mysql> grant all on test.* to redis@'%' identified by 'westos';
Query OK, 0 rows affected (0.00 sec)
测试 ,访问 http://172.25.32.2/test.php
redis master和slave查看
[root@server4 ~]# redis-cli
127.0.0.1:6379> get 1
"test1"
127.0.0.1:6379> exit
[root@server3 redis]# redis-cli
127.0.0.1:6379> get 1
"test1"
127.0.0.1:6379> exit
以上方式 redis 不能同步 mysql;只有 master 端删除原始的 key-value 才能刷新
##数据库更改数据
[root@server3 redis]# mysql
mysql> update test.test set name='redhat' where id=1;
mysql> select * from test.test;
+----+--------+
| id | name |
+----+--------+
| 1 | redhat |
| 2 | test2 |
| 3 | test3 |
| 4 | test4 |
| 5 | test5 |
| 6 | test6 |
| 7 | test7 |
| 8 | test8 |
| 9 | test9 |
+----+--------+
##但是redis还是原始值
[root@server3 redis]# redis-cli
127.0.0.1:6379> get 1
"test1"
127.0.0.1:6379> exit
三、优化
1、worker(server2)
##安装 gearmand
[root@server2 redis]# yum install -y gearmand-1.1.8-2.el6.x86_64.rpm libevent-1.4.13-4.el6.x86_64.rpm libevent-devel-1.4.13-4.el6.x86_64.rpm libevent-doc-1.4.13-4.el6.noarch.rpm libevent-headers-1.4.13-4.el6.noarch.rpm libgearman-1.1.8-2.el6.x86_64.rpm libgearman-devel-1.1.8-2.el6.x86_64.rpm
[root@server2 redis]# /etc/init.d/gearmand start
Starting gearmand: [ OK ]
[root@server2 redis]# netstat -antpl | grep gearmand
tcp 0 0 0.0.0.0:4730 0.0.0.0:* LISTEN 3694/gearmand
tcp 0 0 :::4730 :::* LISTEN 3694/gearmand
##配置 work.php
[root@server2 redis]# vim worker.php
$redis = new Redis();
$redis->connect('172.25.32.4', 6379);
[root@server2 redis]# cp worker.php /usr/local/
##添加 gearman 模块
[root@server2 redis]# tar zxf gearman-1.1.2.tgz
[root@server2 redis]# cd gearman-1.1.2
[root@server2 gearman-1.1.2]# phpize
Configuring for:
PHP Api Version: 20090626
Zend Module Api No: 20090626
Zend Extension Api No: 220090626
[root@server2 gearman-1.1.2]# ./configure
[root@server2 gearman-1.1.2]# make && make install
[root@server2 gearman-1.1.2]# cd /etc/php.d/
[root@server2 php.d]# cp redis.ini gearman.ini
[root@server2 php.d]# vim gearman.ini
[root@server2 php.d]# cat gearman.ini
extension=gearman.so
[root@server2 php.d]# /etc/init.d/php-fpm reload
Reloading php-fpm: [12-Jul-2018 14:44:53] NOTICE: configuration file /etc/php-fpm.conf test is successful
[ OK ]
[root@server2 php.d]# php -m | grep gearman
gearman
2、redis master(server4)不做改动
3、mysql(server3)
##配置 json 函数
[root@server3 redis]# yum install -y mysql-devel gcc unzip
[root@server3 redis]# unzip lib_mysqludf_json-master.zip
[root@server3 redis]# cd lib_mysqludf_json-master
[root@server3 lib_mysqludf_json-master]# gcc $(mysql_config --cflags) -shared -fPIC -o lib_mysqludf_json.so lib_mysqludf_json.c
[root@server3 lib_mysqludf_json-master]# cp lib_mysqludf_json.so /usr/lib64/mysql/plugin/
[root@server3 lib_mysqludf_json-master]# cd
[root@server3 ~]# mysql
mysql> show global variables like 'plugin_dir';
+---------------+-------------------------+
| Variable_name | Value |
+---------------+-------------------------+
| plugin_dir | /usr/lib64/mysql/plugin |
+---------------+-------------------------+
1 row in set (0.00 sec)
mysql> CREATE FUNCTION json_object RETURNS STRING SONAME 'lib_mysqludf_json.so';
Query OK, 0 rows affected (0.00 sec)
mysql> select * from mysql.func;
+-------------+-----+----------------------+----------+
| name | ret | dl | type |
+-------------+-----+----------------------+----------+
| json_object | 0 | lib_mysqludf_json.so | function |
+-------------+-----+----------------------+----------+
1 row in set (0.00 sec)
##配置 udf 函数
[root@server3 redis]# tar zxf gearman-mysql-udf-0.6.tar.gz
[root@server3 redis]# cd gearman-mysql-udf-0.6
[root@server3 gearman-mysql-udf-0.6]# ./configure --libdir=/usr/lib64/mysql/plugin/
##依赖性
[root@server3 redis]# yum install -y libevent.x86_64 0:1.4.13-4.el6 libevent-devel.x86_64 0:1.4.13-4.el6 libevent-doc.noarch 0:1.4.13-4.el6 libevent-headers.noarch 0:1.4.13-4.el6 libgearman.x86_64 0:1.1.8-2.el6 libgearman-devel.x86_64 0:1.1.8-2.el6
[root@server3 gearman-mysql-udf-0.6]# ./configure --libdir=/usr/lib64/mysql/plugin/
[root@server3 gearman-mysql-udf-0.6]# make && make install
[root@server3 ~]# mysql
mysql> CREATE FUNCTION gman_do_background RETURNS STRING SONAME 'libgearman_mysql_udf.so';
Query OK, 0 rows affected (0.00 sec)
mysql> CREATE FUNCTION gman_servers_set RETURNS STRING SONAME 'libgearman_mysql_udf.so';
Query OK, 0 rows affected (0.00 sec)
mysql> select * from mysql.func;
+--------------------+-----+-------------------------+----------+
| name | ret | dl | type |
+--------------------+-----+-------------------------+----------+
| json_object | 0 | lib_mysqludf_json.so | function |
| gman_do_background | 0 | libgearman_mysql_udf.so | function |
| gman_servers_set | 0 | libgearman_mysql_udf.so | function |
+--------------------+-----+-------------------------+----------+
mysql> SELECT gman_servers_set('172.25.120.2:4730');
+---------------------------------------+
| gman_servers_set('172.25.120.2:4730') |
+---------------------------------------+
| 172.25.120.2:4730 |
+---------------------------------------+
##导入触发器
[root@server3 redis]# vim test.sql
[root@server3 redis]# cat test.sql
use test;
#CREATE TABLE `test` (`id` int(7) NOT NULL AUTO_INCREMENT, `name` char(8) DEFAULT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;
#INSERT INTO `test` VALUES (1,'test1'),(2,'test2'),(3,'test3'),(4,'test4'),(5,'test5'),(6,'test6'),(7,'test7'),(8,'test8'),(9,'test9');
DELIMITER $$
CREATE TRIGGER datatoredis AFTER UPDATE ON test FOR EACH ROW BEGIN
SET @RECV=gman_do_background('syncToRedis', json_object(NEW.id as `id`, NEW.name as `name`));
END$$
DELIMITER ;
[root@server3 redis]# mysql < test.sql
[root@server3 redis]# mysql
mysql> use test;
mysql> SHOW TRIGGERS FROM test;
------------------+
| datatoredis | UPDATE | test | BEGIN
SET @RECV=gman_do_background('syncToRedis', json_object(NEW.id as `id`, NEW.name as `name`));
END | AFTER | NULL | | root@localhost | latin1 | latin1_swedish_ci | latin1_swedish_ci |
+-------------+--------+-------+-------------------------------
4、测试
##worker
[root@server2 php.d]# nohup php /usr/local/worker.php &
[1] 6047
[root@server2 php.d]# nohup: ignoring input and appending output to `nohup.out'
##mysal
[root@server3 redis]# mysql
mysql> update test.test set name='redhat' where id=2;
Query OK, 1 row affected (0.21 sec)
[root@server3 redis]# redis-cli
127.0.0.1:6379> get 2
"redhat"
127.0.0.1:6379>
redis 缓存即时同步
访问172.25.32.2/test.php