codis高可用集群搭建

一、集群规划

192.168.66.120:18080 codis-dashboard
192.168.66.120:9090 codis-fe
192.168.66.120:6363 master
192.168.66.121:6363 slave
192.168.99.9:19000 proxy
192.168.99.10 :19000 proxy
192.168.99.217 :19000 proxy
192.168.99.9:23232 sentinel
192.168.99.10:23232 sentinel
192.168.99.217:23232 sentinel
192.168.66.122:2181 zookeeper
192.168.66.123:2181 zookeeper
192.168.66.124:2181 zookeeper

二、CODIS编译安装

所有节点执行以下脚本

#!/bin/bash
echo -e "export PATH=$PATH:/usr/local/go/bin\nexport GOPATH=/usr/local/gocode" >> /etc/profile
source /etc/profile
yum install autoconf automake libtool -y
mkdir -p /usr/local/gocode/src/github.com/CodisLabs
cd /usr/local/gocode/src/github.com/CodisLabs
yum install git -y
yum install go -y
git clone git://github.com/CodisLabs/codis.git -b release3.2
cd codis
make

三、启动DASHBOARD和FE

在192.168.66.120上做如下操作：

1、启动dashboard

修改dashboard配置文件：

coordinator_name = "zookeeper"
coordinator_addr = "192.168.66.122:2181,192.168.66.123:2181,192.168.66.124:2181"

product_name = "CESHI"
product_auth = ""
admin_addr = "192.168.66.120:18080"

# Set arguments for data migration (only accept 'sync' & 'semi-async').
migration_method = "semi-async"
migration_parallel_slots = 100
migration_async_maxbulks = 200
migration_async_maxbytes = "32mb"
migration_async_numkeys = 500
migration_timeout = "30s"

# Set configs for redis sentinel.
sentinel_client_timeout = "10s"
sentinel_quorum = 2
sentinel_parallel_syncs = 1
sentinel_down_after = "30s"
sentinel_failover_timeout = "5m"
sentinel_notification_script = ""
sentinel_client_reconfig_script = ""

启动dashboard：

nohup ./bin/codis-dashboard --ncpu=4 --config=/data/codis/conf/dashboard.toml --log=/data/codis/logs/dashboard.log --log-level=WARN &

#关闭dashboard命令：

./codis-admin --dashboard=192.168.66.120:18080 --shutdown

2、启动fe

拉取codis.json

/usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-admin  --dashboard-list --zookeeper=192.168.66.122:2181,192.168.66.123:2181,192.168.66.124:2181 | tee codis.json
mv codis.json /data/codis/conf/

启动fe

nohup  /usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-fe --log=/data/codis/logs/fe.log --log-level=WARN --dashboard-list=/data/codis/conf/codis.json --listen=192.168.66.120:9090 &

四、启动Codis Master和Slave

1、在192.168.66.120上启动master

编辑master配置文件

bind 192.168.66.120
port 6363
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile "/data/codis/redis/redis_6363/redis_6363.pid"
loglevel notice
logfile "/data/codis/redis/redis_6363/redis_6363.log"
databases 16
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename "redis_6363.rdb"
dir "/data/codis/redis/redis_6363"
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
maxmemory 10gb
maxmemory-policy allkeys-lru
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 100
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
# Generated by CONFIG REWRITE

启动master

/usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-server /data/codis/redis/redis_6363/redis.conf

2、在192.168.66.121上启动slave

编辑slave配置文件，以下三行与master不同，其余相同

bind 192.168.66.121
port 6363
slaveof 192.168.66.120 6363   ##最后一行增加master信息

启动slave

/usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-server /data/codis/redis/redis_6363/redis.conf

五、启动proxy

在proxy三台机器上做如下操作：
编辑配置文件：

##################################################
#                                                #
#                  Codis-Proxy                   #
#                                                #
##################################################

# Set Codis Product Name/Auth.
product_name = "CESHI"
product_auth = ""

# Set auth for client session
#   1. product_auth is used for auth validation among codis-dashboard,
#      codis-proxy and codis-server.
#   2. session_auth is different from product_auth, it requires clients
#      to issue AUTH <PASSWORD> before processing any other commands.
session_auth = ""

# Set bind address for admin(rpc), tcp only.
admin_addr = "192.168.99.217:11080"

# Set bind address for proxy, proto_type can be "tcp", "tcp4", "tcp6", "unix" or "unixpacket".
proto_type = "tcp4"
proxy_addr = "0.0.0.0:19000"

# Set jodis address & session timeout
#   1. jodis_name is short for jodis_coordinator_name, only accept "zookeeper" & "etcd".
#   2. jodis_addr is short for jodis_coordinator_addr
#   3. jodis_auth is short for jodis_coordinator_auth, for zookeeper/etcd, "user:password" is accepted.
#   4. proxy will be registered as node:
#        if jodis_compatible = true (not suggested):
#          /zk/codis/db_{PRODUCT_NAME}/proxy-{HASHID} (compatible with Codis2.0)
#        or else
#          /jodis/{PRODUCT_NAME}/proxy-{HASHID}
jodis_name = "zookeeper"
jodis_addr = "192.168.66.122:2181,192.168.66.123:2181,192.168.66.124:2181"
jodis_auth = ""
jodis_timeout = "20s"
jodis_compatible = false

# Set datacenter of proxy.
proxy_datacenter = ""

# Set max number of alive sessions.
proxy_max_clients = 10000

# Set max offheap memory size. (0 to disable)
proxy_max_offheap_size = "1024mb"

# Set heap placeholder to reduce GC frequency.
proxy_heap_placeholder = "256mb"

# Proxy will ping backend redis (and clear 'MASTERDOWN' state) in a predefined interval. (0 to disable)
backend_ping_period = "5s"

# Set backend recv buffer size & timeout.
backend_recv_bufsize = "128kb"
backend_recv_timeout = "180s"

# Set backend send buffer & timeout.
backend_send_bufsize = "128kb"
backend_send_timeout = "180s"

# Set backend pipeline buffer size.
backend_max_pipeline = 20480

# Set backend never read replica groups, default is false
backend_primary_only = false

# Set backend parallel connections per server
backend_primary_parallel = 1
backend_replica_parallel = 1

# Set backend tcp keepalive period. (0 to disable)
backend_keepalive_period = "75s"

# Set number of databases of backend.
backend_number_databases = 16

# If there is no request from client for a long time, the connection will be closed. (0 to disable)
# Set session recv buffer size & timeout.
session_recv_bufsize = "128kb"
session_recv_timeout = "30m"

# Set session send buffer size & timeout.
session_send_bufsize = "64kb"
session_send_timeout = "30s"

# Make sure this is higher than the max number of requests for each pipeline request, or your client may be blocked.
# Set session pipeline buffer size.
session_max_pipeline = 10000

# Set session tcp keepalive period. (0 to disable)
#session_keepalive_period = "75s"
session_keepalive_period = 0

# Set session to be sensitive to failures. Default is false, instead of closing socket, proxy will send an error response to client.
session_break_on_failure = false

# Set metrics server (such as http://localhost:28000), proxy will report json formatted metrics to specified server in a predefined period.
metrics_report_server = ""
metrics_report_period = "1s"

# Set influxdb server (such as http://localhost:8086), proxy will report metrics to influxdb.
#metrics_report_influxdb_server = "http://1*:*"
#metrics_report_influxdb_period = "5s"
#metrics_report_influxdb_username = ""
#metrics_report_influxdb_password = ""
#metrics_report_influxdb_database = ""

# Set statsd server (such as localhost:8125), proxy will report metrics to statsd.
metrics_report_statsd_server = ""
metrics_report_statsd_period = "1s"
metrics_report_statsd_prefix = ""

启动proxy

nohup /usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-proxy  --config=/data/codis/conf/proxy.toml  --log=/data/codis/logs/proxy.log --log-level=WARN &

#关闭proxy

/usr/local/gocode/src/github.com/CodisLabs/codis/bin/codis-admin --proxy=192.168.99.10:21080 --auth=ceshi --shutdown

六、启动哨兵

在哨兵三台机器上做如下操作：
编辑配置文件：

daemonize yes
bind 192.168.99.217    ##sentinel所在机器IP
port 23232			   ##sentinel端口
logfile "/data/codis/sentinel/sentinel.log"
dir "/data/codis/sentinel"

启动哨兵

 /usr/local/gocode/src/github.com/CodisLabs/codis/bin/redis-sentinel /data/codis/sentinel/sentinel.conf

#关闭哨兵

 /usr/local/gocode/src/github.com/CodisLabs/codis/bin/redis-cli -h 192.168.99.9 -p 23232 shutdown

七、登陆FE添加集群

登陆http://192.168.66.120:9090/
按照配置的IP:端口添加PROXY、SERVER、SENTINEL即可。
注：SENTINEL添加完成后需点击rsync按钮同步集群信息