redis集群部署之codis 维护脚本

 

搞了几天redis cluster codis 的部署安装，测试，架构优化，配合研发应用整合，这里记一些心得！

背景需求：

之前多个业务都在应用到redis库，各业务独立占用主从两台服务器，硬件资源利用不合理，主从架构冗余度不高，主redis故障的话，从redis恢复需要时间，降低业务的可用性， 所以调研测试部署了基于codis的redis集群。 官方地址

部署文档 参考这里http://www.cnblogs.com/shantu/p/4589798.html

维护管理

了解过codis的同志都知道codis集群组件服务启动有一定顺序的，而且基本了 添加组，初始化slot ,各节点启动proxy都依懒于dashboard, 所以要遵循以下基本步骤。

• start dashboard
• start master and slave codis redis
• add group
• init slot
• start proxy

下面根据自已的使用习惯写了一个codis集群的管理脚本

• 主要实现codis集群服务组件启动关闭、redis配置文件生成的功能，python学的实在太烂，之前写的那个更烂，这个做了些优化
• 使用说明:codis 本身默认配置为config.ini 放在当前home, 使用此脚本，也需要预配codis.ini配置文件，供脚本解析调用，

如下:

[codis_group]
group_1 = 192.168.5.14:6381 master,192.168.5.15:6381 slave
group_2 = 192.168.5.15:6382 master,192.168.5.14:6382 slave
group_3 = 192.168.5.16:6383 master,192.168.5.44:6383 slave
group_4 = 192.168.5.44:6384 master,192.168.5.16:6384 slave

• cat redis_manage.py

        #!/usr/bin/env python
        # coding:utf8
        #author:shantuwqk@163.com


        import os,sys,commands,time import ConfigParser from subprocess import Popen,PIPE from mako.template import Template codis_config = "config.ini" codis_root = "/data/setup/codis/data" codis_dbdir = "%s/db"%codis_root def pars_config(): codis_group_dict = {} cf = ConfigParser.ConfigParser() cf.read("codis.ini") codis_group = cf.items("codis_group") return codis_group def codis_dashboard(opt): if opt == "start": #os.chdir(codis_root) exec_cmd = "cd %s; nohup ../bin/codis-config -c %s -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/http.log &>/dev/null &" %(codis_root,codis_config) s = os.system(exec_cmd) if s == 0: print "\033[32;1m codis dashboard start .... OK\033[0m" else: print "\033[31;1m codis dashboard start .... Error\033[0m" elif opt == "stop": dashboard_id = "ps aux |grep \"codis-config\"| grep \"dashboard\"|awk '{print $2}'" s,v = commands.getstatusoutput("kill -9 `%s`" %dashboard_id) if s == 0: print "\033[32;1m KILL codis dashboard id:[%s] OK\033[0m"%dashboard_id else: print "\033[31;1m KILL codis dashboard id:[%s] Error\033[0m"%dashboard_id else: pass def INIT_config_master(port,memsize,datadir,gtag): os.chdir(codis_root) template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir) conf = open('./conf/%s.conf'%gtag,'w') conf.write(template_file) conf.close() dirisnot = os.path.exists(datadir) redisconf = os.path.isfile("%s/conf/%s.conf"%(codis_root,gtag)) if not dirisnot and redisconf: print dirisnot,"create %s ...."%datadir os.makedirs(datadir) print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(codis_root,gtag))[1] return True else: print "%s is exists !!"%datadir return False def INIT_config_slave(port,memsize,datadir,slaveof,gtag): os.chdir(codis_root) template_file = Template(filename="./conf/redis.slave.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir,slaveof="%s %s"%(slaveof.split(":")[0],slaveof.split(":")[1])) conf = open('./conf/%s.conf'%gtag,'w') conf.write(template_file) conf.close() dirisnot = os.path.exists(datadir) redisconf = os.path.isfile("./conf/%s.conf"%(gtag)) if not dirisnot and redisconf: print dirisnot,"create %s ...."%datadir os.makedirs(datadir) print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(codis_root,gtag))[1] return True else: print "%s is exists !!"%datadir return False def codis_redis(opt,cf): print "###################33",cf, cname = cf[:-5] print cname, port = cf[-9:-5] print "#########",port, redis_id = "ps aux | grep codis-server| grep -v grep |grep %s|awk '{print $2}'"%port print redis_id if opt == "start": ids,idv = commands.getstatusoutput(redis_id) print ids,type(idv) if ids == 0 and idv != "": print "\033[32;1m%s redis process Already exists pid is:[%s]\033[0m"%(port,idv) sys.exit(1) else: exec_cmd = "cd %s; nohup ../bin/codis-server ./conf/%s.conf &> ./log/%s.log &" %(codis_root,cname,cname) print exec_cmd print os.system(exec_cmd) s = os.system(exec_cmd) if s == 0: print "\033[32;1m start redis %s server is OK \033[0m"%(port) else: print "\033[31;1m start redis %s server is Error\033[0m"%(port) elif opt == "stop": s,v = commands.getstatusoutput("kill -9 `%s`" %redis_id) if s == 0: print "\033[32;1m KILL redis port:[%s] pid:[%s] is OK\033[0m"%(port,redis_id) print v, else: print "\033[31;1m KILL redis port:[%s] pid:[%s] is Error\033[0m"%(port,redis_id) print v, else: pass def codis_group(): print "change codis.ini exists?" if os.path.isfile("codis.ini") and pars_config() != "": print "\033[32;1mparsing the codis.ini configuration file\033[0m" for i in sorted(pars_config()): #print i[1],'\n' for j in i[1].split(","): #print j,'\n', exec_cmd = "cd %s; ../bin/codis-config -c %s -L ./log/%s_addgroup.log server add %s %s %s" %(codis_root,codis_config,j.split()[0].split(":")[1],i[0].split("_")[1],j.split()[0],j.split()[1]) print exec_cmd,'\n', s,v = commands.getstatusoutput(exec_cmd) print s,v, if s == 0: print "\033[32;1m add group:[%s] with a gtag:[%s] addr:(%s) OK\033[0m" %(i[0].split("_")[1],j.split()[1],j.split()[0]) else: print "\033[31;1m add group:[%s] with a gtag:[%s] addr:(%s) Error\033[0m" %(i[0].split("_")[1],j.split()[1],j.split()[0]) else: print "config not exist!!" def remove_fenc(): remove_fenc = "cd %s;../bin/codis-config -c %s action remove-fence"%(codis_root,codis_config) s,v = commands.getstatusoutput(remove_fenc) print "remove fenc proxy info",v, def slot_init(): init_cmd = "cd %s; ../bin/codis-config -c %s slot init -f" %(codis_root,codis_config) print "INIT SLOT ........","\n",commands.getstatusoutput(init_cmd)[1] def codis_initslot(gid,slot_range): exec_cmd = "cd %s; ../bin/codis-config -c %s slot range-set %s %s %s online"%(codis_root,codis_config,slot_range[0],slot_range[1],gid) print exec_cmd s,v = commands.getstatusoutput(exec_cmd) print v, if s == 0: print "\033[32;1m slot init:[%s],gid:[%s] OK\033[0m"%(slot_range,gid) else: print "\033[31;1m slot init:[%s],gid:[%s] Error\033[0m"%(slot_range,gid) def offline_proxy(): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print "Shutdown %s offline....." %proxy_tag print exec_down_cmd commands.getstatusoutput(exec_down_cmd) def codis_proxy(opt): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] if opt == "start": exec_new_cmd = "cd %s; nohup ../bin/codis-proxy --log-level info -c %s -L ./log/%s.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &" %(codis_root,codis_config,proxy_tag) print exec_new_cmd s = os.system(exec_new_cmd) if s == 0: print "\033[32;1m codis proxy tag:[%s]start OK\033[0m"%(proxy_tag) else: print "\033[31;1m codis proxy tag:[%s]start Error\033[0m"%(proxy_tag) time.sleep(5) exec_online_cmd = "cd %s;../bin/codis-config -c %s proxy online %s" %(codis_root,codis_config,proxy_tag) print "Set %s online .....!!" %(proxy_tag) print exec_online_cmd print commands.getstatusoutput(exec_online_cmd)[1] elif opt == "stop": print "Shutdown %s offline....." %proxy_tag exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print commands.getstatusoutput(exec_down_cmd)[1] exec_proxy_id = "ps aux |grep codis-proxy| grep -v grep|awk '{print $2}'" print commands.getstatusoutput("kill -9 `%s`"%exec_proxy_id) def get_client_ip(): eth_inter= commands.getstatusoutput("ifconfig -a| awk '/^em/ {;a=$1;FS=\":\"; nextline=NR+1; next}{ if (NR==nextline) { split($2,b,\" \")}{ if ($2 ~ /[0-9]\./) {print a,b[1]}; FS=\" \"}}'|uniq -c|awk '{print $2,$3}'") if eth_inter[0] == 0: return eth_inter[1].split()[1] else: print "get client ip error" def slot_range(n,group): slot_dict = {} num=0 l = range(n) per = len(l) / group #改变i的索引值 for i in l[::per]: avg=l[i:i+per] tupv01 = avg[0],avg[-1] #print type(tupv01),tupv01 #指定当前添加的组数 num+=1 slot_dict[num] = [] slot_dict[num].append(tupv01) #如果只剩下一组，则将剩余元素全部追加至列表 if num==group-1: tupv02 = l[i+per:][0],l[i+per:][-1] #print tupv02 v02key = num + 1 slot_dict[v02key] = [] slot_dict[v02key].append(tupv02) break #for k,port in redis_instance_port.items(): # if k in slot_dict.keys(): # slot_dict[k].append(port) #print slot_dict return slot_dict def handle_slot(): n = len(pars_config()) return slot_range(1024,n) def init_config(): for i in sorted(pars_config()): for j in i[1].split(","): if j.split()[1] == "master": confname = i[0] + "_" + j.split()[1] + "_" + j.split()[0].split(":")[0] + "_" + j.split()[0].split(":")[1] INIT_config_master(j.split()[0].split(":")[1],6,codis_dbdir,confname) mastername = j.split()[0] elif j.split()[1] == "slave": confname = i[0] + "_" + j.split()[1] + "_" + j.split()[0].split(":")[0] + "_" + j.split()[0].split(":")[1] INIT_config_slave(j.split()[0].split(":")[1],6,codis_dbdir,mastername,confname) else: pass def start_master(): mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"master\"|awk '{print $9}'"%(codis_root,get_client_ip()) print mcfile_cmd mcfile = commands.getstatusoutput(mcfile_cmd)[1] codis_redis('start',mcfile) def stop_master(): mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"master\"|awk '{print $9}'"%(codis_root,get_client_ip()) print mcfile_cmd mcfile = commands.getstatusoutput(mcfile_cmd)[1] codis_redis('stop',mcfile) def start_slave(): mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"slave\"|awk '{print $9}'"%(codis_root,get_client_ip()) print mcfile_cmd mcfile = commands.getstatusoutput(mcfile_cmd)[1] codis_redis('start',mcfile) def stop_slave(): mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"slave\"|awk '{print $9}'"%(codis_root,get_client_ip()) print mcfile_cmd mcfile = commands.getstatusoutput(mcfile_cmd)[1] codis_redis('stop',mcfile) def assign_slot(): remove_fenc() slot_init() for k,v in handle_slot().items(): codis_initslot(k,v[0]) def help_prompt(): print """ This program prints files to the standard output. Options include: --version : Prints the version number --help : Helpful tips sample : python codis_manage.py init : python codis_manage.py start_master/stop_master start_slave/stop_slave start_web/stop_web add_group initslot start_proxy/stop_proxy """ if __name__ == "__main__": if len(sys.argv) < 2: print "no argument",'\n',help_prompt() sys.exit(1) if sys.argv[1] == "--help": help_prompt() elif sys.argv[1] == "--version": print "Version 0.1" elif sys.argv[1] == "init": init_config() elif sys.argv[1] == "start_master": start_master() elif sys.argv[1] == "stop_master": stop_master() elif sys.argv[1] == "start_slave": start_slave() elif sys.argv[1] == "stop_slave": stop_slave() elif sys.argv[1] == "start_web": codis_dashboard('start') elif sys.argv[1] == "stop_web": codis_dashboard('stop') elif sys.argv[1] == "add_group": codis_group() elif sys.argv[1] == "initslot": assign_slot() elif sys.argv[1] == "start_proxy": codis_proxy('start') elif sys.argv[1] == "stop_proxy": codis_proxy('stop') elif sys.argv[1] == "stopall": codis_proxy('stop') commands.getstatusoutput("killall codis-server") codis_dashboard("stop") else: help_prompt() print get_client_ip() 

这里还有一个基于redis sentinel方式的集群redis服务启停脚本

• sentinel.conf配置示例

 port 26330
sentinel monitor lashou 192.168.5.14 6479 2
sentinel down-after-milliseconds lashou 3000
sentinel failover-timeout lashou 4000
sentinel notification-script lashou /data/setup/redis-2.8.19/bin/get_redis_master.py

 cat get_redis_master.py

#!/usr/bin/env python # coding:utf8 #author:shantuwqk@163.com import os,sys,commands,time from subprocess import Popen,PIPE from mako.template import Template from get_redis_master import get_redis_master_info REDIS_ROOT = "/data/setup/redis-2.8.19" REDIS_DBDIR = "%s/data"%REDIS_ROOT def INIT_config_master(port,memsize,datadir): os.chdir(REDIS_ROOT) template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir) conf = open('./conf/redis_%s.conf'%port,'w') conf.write(template_file) conf.close() dirisnot = os.path.exists(datadir) redisconf = os.path.isfile("%s/conf/redis_%s.conf"%(REDIS_ROOT,port)) if not dirisnot and redisconf: print dirisnot,"create %s ...."%datadir os.makedirs(datadir) print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(REDIS_ROOT,port))[1] return True else: print "%s is exists !!"%datadir return False def INIT_config_slave(port,memsize,datadir,slaveof): os.chdir(REDIS_ROOT) template_file = Template(filename="./conf/redis.slave.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir,slaveof="%s %s"%(slaveof.split(":")[0],slaveof.split(":")[1])) conf = open('./conf/redis_%s.conf'%port,'w') conf.write(template_file) conf.close() dirisnot = os.path.exists(datadir) redisconf = os.path.isfile("%s/conf/redis_%s.conf"%(REDIS_ROOT,port)) if not dirisnot and redisconf: print dirisnot,"create %s ...."%datadir os.makedirs(datadir) print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(REDIS_ROOT,port))[1] return True else: print "%s is exists !!"%datadir return False def start_redis(redis_root,port): start_redis_cmd = "cd %s ; nohup ./src/redis-server conf/redis_%s.conf &> ./logs/%s_redis.log & "%(REDIS_ROOT,port,port) s = os.system(start_redis_cmd) if s == 0: print " redis start instence:[%s] ok"%(port) else: print " redis start instence:[%s] faild"%(port) def stop_redis(port): stop_redis_pid = "ps aux | grep redis| grep %s|awk '{print $2}'"%(port) stop_redis_cmd = "kill -9 `%s`"%stop_redis_pid s,v = commands.getstatusoutput(stop_redis_cmd) if s == 0: print " redis stop instence:[%s] ok"%(port),"\n",v else: print " redis stop instence:[%s] faild"%(port) def help_prompt(): print """ This program prints files to the standard output. Options include: --version : Prints the version number --help : Helpful tips --task : To operate on missions: [start_redis|stop_redis] [port] sample : python redis_manage.py start/stop master/slave port """ if len(sys.argv) < 2: print "no argument" help_prompt() sys.exit() if sys.argv[1].startswith('--'): option = sys.argv[1][2:] if option == 'version': print 'Version 0.1' elif option == 'help': help_prompt() elif sys.argv[1] == "start" and sys.argv[2] == "master" and sys.argv[3] is not None: INIT_config_master(sys.argv[3],6,REDIS_DBDIR) time.sleep(2) start_redis(REDIS_ROOT,sys.argv[3]) elif sys.argv[1] == "start" and sys.argv[2] == "slave" and sys.argv[3] is not None: INIT_config_slave(sys.argv[3],6,REDIS_DBDIR,get_redis_master_info()) time.sleep(2) start_redis(REDIS_ROOT,sys.argv[3]) elif sys.argv[1] == "stop" and sys.argv[2] is not None: print commands.getstatusoutput("ps aux | grep redis-server|grep -v grep")[1] stop_redis(sys.argv[2]) else: help_prompt() 

触发更新haproxy配置文件的脚本

 #!/usr/bin/env python
# coding:utf8
#author:shantuwqk@163.com


import os,sys,commands,time from subprocess import Popen,PIPE from mako.template import Template #init_redis_master = REDIS_ROOT = "/data/setup/redis-2.8.19" def get_redis_master_info(): cmd = "cd %s;./src/redis-cli -h 192.168.5.15 -p 26329 sentinel masters| grep -A 4 \"ip\"|head -4"%REDIS_ROOT s,v = commands.getstatusoutput(cmd) #print v, redis_master = "%s:%s"%(v.split("\n")[1],v.split("\n")[3]) return redis_master def handle_redis_m(): redis_master_dict = {} redis_master_dict[get_redis_master_info()] = "redis_master_%s_%s"%(get_redis_master_info().split(":")[0].split(".")[3],get_redis_master_info().split(":")[1]) return redis_master_dict def handle_redis_s(): redis_slave_dict = {} cmd = "cd %s;./src/redis-cli -h 192.168.5.15 -p 26329 sentinel slaves lashou| grep -A 1 \"name\"| grep -v \"\-\-\\|name\""%REDIS_ROOT s,v = commands.getstatusoutput(cmd) slave_list = v.split("\n") for i in slave_list: redis_slave_dict[i] = "redis_slave_%s_%s"%(i.split(":")[0].split(".")[3],i.split(":")[1]) return redis_slave_dict def update_ha_conf(redis_slave_dict,redis_master_dict): os.chdir(REDIS_ROOT) template_file = Template(filename="./conf/haproxy.cfg.template",module_directory='tmp/test').render(redis_slave_dict=redis_slave_dict,redis_master_dict=redis_master_dict) conf = open('./conf/haproxy.cfg','w') conf.write(template_file) conf.close() haproxy_conf = os.path.isfile("%s/conf/haproxy.cfg"%(REDIS_ROOT)) if haproxy_conf: print "update haproxy config info master:[%s] and slave:[%s] ...."%(redis_master_dict,redis_slave_dict) print commands.getstatusoutput("ls -l %s/conf/haproxy.cfg"%(REDIS_ROOT))[1] return True else: print "%s is not exists !!"%haproxy_conf return False remote_ha_root = "/data/setup/haproxy/conf" localhafile = "/data/setup/redis/conf/haproxy.cfg" def reload_ha_service(addr): rsync_cmd = "rsync -avz -e \"ssh -p 22\" %s root@%s:%s/"%(localhafile,addr,remote_ha_root) remote_cmd = "ssh -p 22 root@%s \" /usr/local/sbin/haproxy -f /data/setup/haproxy-1.5.10/conf/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)\"" rs,rv = commands.getstatusoutput(rsync_cmd) remotes,remotev = commands.getstatusoutput(remote_cmd) if rs == 0 and remotes == 0: print "haproxy.cfg update ok" else: print "haproxy.cfg update faild!!" time.sleep(5) if __name__ == "__main__": print get_redis_master_info() print update_ha_conf(handle_redis_s(),handle_redis_m()) 

如果需要迁移现有 redis 数据到 codis，该如何操作？

• 先搭建好 codis 集群并让 codis-proxy 正确运行起来
• 对线上每一个 redis 实例运行一个 redis-port 来向 codis 导入数据，例如：

for port in {6379,6380,6479,6480}; do
    nohup redis-port sync --ncpu=4 --from=redis-server:${port} \
        --target=codis-proxy:19000 > ${port}.log 2>&1 &
    sleep 5
done
tail -f *.log

 每个 redis-port 负责将对应的 redis 数据导入到 codis
 多个 redis-port 之间不互相干扰，除非多个 redis 上的 key 本身出现冲突
单个 redis-port 可以将负责的数据并行迁移以提高速度，通过 --ncpu 来指定并行数
导入速度受带宽以及 codis-proxy 处理速度限制(本质是大量的 slotsrestore 操作)
完成数据迁移，在适当的时候将服务指向 Codis，并将原 redis 下线

• 旧 redis 下线时，会导致 reids-port 链接断开，于是自动退出

转载于:https://www.cnblogs.com/shantu/p/4586697.html