redis cluster安装部署(测试环境)

 

redis 应用于web前端，做缓存和数据存取的速度是挺可观的，最近看了一些资料，手痒了，就弄了一个测试环境，两台方案，试用一下。

##Redis 集群部署##

一，方案调研:

参考博客:

http://jolestar.com/redis-ha/

http://www.luocs.com/archives/tag/redis

https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md

https://github.com/twitter/twemproxy

二，部署架构

三,方案简介：

需求:

    缓存:定时任务临时写入和读取的数据量大概在10G左右
    存储:日常存储量大概在40G左右
    需要7＊24小时不间断提供数据服务，所以要求redis集群要稳定，在有故障时可自动切换和数据即时恢复，并在业务增长或下降时，集群可伸缩或扩展（手动），并不影响在线业务，故调研测试这两种方案，以codis 为代理的codis server redis的集群，以twemproxy为代理的redis集群
codis 集群技术原理参考  https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md
twemproxy 集群技术原理参考:
https://github.com/twitter/twemproxy
http://blog.jpush.cn/redis-twemproxy-benchmark/

四,部署架构规划

服务器操作系统： centos 6.x  注意:centos 5版本的内核支持不了codis 安装编译
服务器部署规划:
192.168.5.14        keepalived ,codis相关服务
192.168.5.15        keepalived,codis相关服务
192.168.5.16        twemporxy,redis,haproxy
192.168.5.44        twemporxy,redis,haproxy

五,部署安装

1,codis 方案安装

1.1安装go 环境:
首先按照golang，下载地址：https://golang.org/dl/，最新的1.4.2版本。 
如果被墙使用golang中国下载http://golangtc.com/download。 
cd /usr/loca/
wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
tar -zxvf go1.4.2.linux-amd64.tar.gz


vim /etc/profile
最后追加
#set go & codis environment
GOROOT=/usr/local/go
PATH=$PATH:$GOROOT/bin
GOPATH=/usr/local/codis
export GOROOT  PATH  GOPATH

source /etc/profile

1.2 安装配置codis 相关配置

cd /usr/local
git clone https://github.com/wandoulabs/codis.git
#将codis的pkg包拷贝到$GOPATH目录。
mkdir -p /usr/local/codis/src/github.com/wandoulabs/codis

cp /usr/local/codis/pkg/ /usr/local/codis/cmd/  /usr/local/codis/src/github.com/wandoulabs/codis -R

cd /usr/local/codis/ ; sh bootstrap.sh   #安装编译需要些时间，过程略过...... 大概3分钟左右，我的是这样，你的环境，哈哈，你懂的，

 1.3配置脚本:

github上原码中有示例，可以参考试用  https://github.com/wandoulabs/codis
在$path/sample 目录下:

cat startall.sh
./start_dashboard.sh

sleep 3

./start_redis.sh

./add_group.sh

./initslot.sh

./start_proxy.sh

./set_proxy_online.sh

cat  start_dashboard.sh


#!/bin/sh

    nohup ../bin/codis-config -c config.ini -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/requests.log &>/dev/null &



启动前注意一下redis的配置文件./redis_conf/*.conf
cat  start_redis.sh



#!/bin/sh



    nohup ../bin/codis-server ./redis_conf/6381.conf &> ./log/redis_6381.log &

    nohup ../bin/codis-server ./redis_conf/6382.conf &> ./log/redis_6382.log &

    echo "sleep 3s"

    sleep 3

    tail -n 30 ./log/redis_6381.log

    tail -n 30 ./log/redis_6382.log


        cat   add_group.sh


    #!/bin/sh

    echo "add group 1 with a master(localhost:6381), Notice: do not use localhost when in produciton"

    ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 5 192.168.5.44:6381 master



    echo "add group 2 with a master(localhost:6382), Notice: do not use localhost when in produciton"

    ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 6 192.168.5.44:6382 master

        cat  ./initslot.sh

        #!/bin/sh

    echo "slots initializing..."

    #../bin/codis-config -c config.ini slot init -f

    echo "done"



    echo "set slot ranges to server groups..."

    ../bin/codis-config -c  config.ini slot range-set 0 511 5 online

    ../bin/codis-config -c  config.ini slot range-set 512 1023 6 online

    echo "done"


        cat  ./start_proxy.sh
        #!/bin/sh

    echo "shut down proxy_g5_g6.."

    ../bin/codis-config -c config.ini proxy offline proxy_g5_g6

    echo "done"



    echo "start new proxy..."

    nohup ../bin/codis-proxy --log-level info -c config.ini -L ./log/proxy.log  --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &

    echo "done"



    echo "sleep 3s"

    sleep 3

    tail -n 30 ./log/proxy.log



        cat  ./set_proxy_online.sh

        #!/bin/sh

        echo "set proxy_1 online"

        ../bin/codis-config -c config.ini proxy online proxy_g5_g6

        echo "done"

1.4有自已编写的一个类上面所有脚本的python脚本：在测试中，

#!/usr/bin/env python
# coding:utf8
#author:shantuwqk@163.com


import os,sys,commands,time from subprocess import Popen,PIPE from mako.template import Template codis_config = "config.ini" codis_root = "/data/setup/codis/data" def codis_dashboard(opt): if opt == "start": #os.chdir(codis_root) exec_cmd = "cd %s; nohup ../bin/codis-config -c %s -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/http.log &>/dev/null &" %(codis_root,codis_config) s = os.system(exec_cmd) if s == 0: print "\033[32;1m codis dashboard start .... OK\033[0m" else: print "\033[31;1m codis dashboard start .... Error\033[0m" elif opt == "stop": dashboard_id = "ps aux |grep \"codis-config\"| grep \"dashboard\"|awk '{print $2}'" s,v = commands.getstatusoutput("kill -9 `%s`" %dashboard_id) if s == 0: print "\033[32;1m KILL codis dashboard id:[%s] OK\033[0m"%dashboard_id else: print "\033[31;1m KILL codis dashboard id:[%s] Error\033[0m"%dashboard_id else: pass def codis_redis(opt,port): if opt == "start": exec_cmd = "cd %s; nohup ../bin/codis-server ./conf/%s.conf &> ./log/%s.log &" %(codis_root,port,port) print exec_cmd print os.system(exec_cmd) s = os.system(exec_cmd) if s == 0: print "\033[32;1m start redis %s server is OK \033[0m"%(port) else: print "\033[31;1m start redis %s server is Error\033[0m"%(port) elif opt == "stop": redis_id = "ps aux | grep codis-server| grep %s|awk '{print $2}'"%port s,v = commands.getstatusoutput("kill -9 `%s`" %redis_id) if s == 0: print "\033[32;1m KILL redis port:[%s] pid:[%s] is OK\033[0m"%(port,redis_id) print v, else: print "\033[31;1m KILL redis port:[%s] pid:[%s] is Error\033[0m"%(port,redis_id) print v, else: pass def codis_group(gid,addr,gtag): exec_cmd = "cd %s; ../bin/codis-config -c %s -L ./log/%s_addgroup.log server add %s %s %s" %(codis_root,codis_config,addr.split(':')[1],gid,addr,gtag) print "start add codis cluster group info.....","\n",exec_cmd s,v = commands.getstatusoutput(exec_cmd) print s,v, if s == 0: print "\033[32;1m add group:[%s] with a gtag:[%s] addr:(%s) OK\033[0m" %(gid,gtag,addr) else: print "\033[31;1m add group:[%s] with a gtag:[%s] addr:(%s) Error\033[0m" %(gid,gtag,addr) def remove_fenc(): remove_fenc = "cd %s;../bin/codis-config -c %s action remove-fence"%(codis_root,codis_config) s,v = commands.getstatusoutput(remove_fenc) print "remove fenc proxy info",v, def slot_init(): init_cmd = "cd %s; ../bin/codis-config -c %s slot init -f" %(codis_root,codis_config) print "INIT SLOT ........","\n",commands.getstatusoutput(init_cmd)[1] def codis_initslot(gid,slot_range): exec_cmd = "cd %s; ../bin/codis-config -c %s slot range-set %s %s %s online"%(codis_root,codis_config,slot_range[0],slot_range[1],gid) print exec_cmd s,v = commands.getstatusoutput(exec_cmd) print v, if s == 0: print "\033[32;1m slot init:[%s],gid:[%s] OK\033[0m"%(slot_range,gid) else: print "\033[31;1m slot init:[%s],gid:[%s] Error\033[0m"%(slot_range,gid) def offline_proxy(): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print "Shutdown %s offline....." %proxy_tag print exec_down_cmd commands.getstatusoutput(exec_down_cmd) def codis_proxy(opt): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] if opt == "start": exec_new_cmd = "cd %s; nohup ../bin/codis-proxy --log-level info -c %s -L ./log/%s.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &" %(codis_root,codis_config,proxy_tag) print exec_new_cmd s = os.system(exec_new_cmd) if s == 0: print "\033[32;1m codis proxy tag:[%s]start OK\033[0m"%(proxy_tag) else: print "\033[31;1m codis proxy tag:[%s]start Error\033[0m"%(proxy_tag) time.sleep(5) exec_online_cmd = "cd %s;../bin/codis-config -c %s proxy online %s" %(codis_root,codis_config,proxy_tag) print "Set %s online .....!!" %(proxy_tag) print exec_online_cmd print commands.getstatusoutput(exec_online_cmd)[1] elif opt == "stop": print "Shutdown %s offline....." %proxy_tag exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print commands.getstatusoutput(exec_down_cmd)[1] exec_proxy_id = "ps aux |grep codis-proxy| grep -v grep|awk '{print $2}'" print commands.getstatusoutput("kill -9 `%s`"%exec_proxy_id) def get_client_ip(): eth_inter= commands.getstatusoutput("ifconfig -a| awk '/^em/ {;a=$1;FS=\":\"; nextline=NR+1; next}{ if (NR==nextline) { split($2,b,\" \")}{ if ($2 ~ /[0-9]\./) {print a,b[1]}; FS=\" \"}}'|uniq -c|awk '{print $2,$3}'") if eth_inter[0] == 0: return eth_inter[1].split()[1] else: print "get client ip error" def slot_range(n,group): slot_dict = {} num=0 l = range(n) per = len(l) / group #改变i的索引值 for i in l[::per]: avg=l[i:i+per] tupv01 = avg[0],avg[-1] #print type(tupv01),tupv01 #指定当前添加的组数 num+=1 slot_dict[num] = [] slot_dict[num].append(tupv01) #如果只剩下一组，则将剩余元素全部追加至列表 if num==group-1: tupv02 = l[i+per:][0],l[i+per:][-1] #print tupv02 v02key = num + 1 slot_dict[v02key] = [] slot_dict[v02key].append(tupv02) break for k,port in redis_instance_port.items(): if k in slot_dict.keys(): slot_dict[k].append(port) return slot_dict def help_prompt(): print """ This program prints files to the standard output. Options include: --version : Prints the version number --help : Helpful tips --task : To operate on missions: [init_codis_cluster] [start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot] sample : python es_manage.py --task init_codis_cluster : python es_manage.py --task [start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot] """ if __name__ == "__main__": if len(sys.argv) < 2: print "no argument" sys.exit() if sys.argv[1].startswith('--'): option = sys.argv[1][2:] if option == 'version': print 'Version 0.1' elif option == 'help': help_prompt() if option == "task" and sys.argv[2] == "init_codis_cluster" and sys.argv[2] is not None: codis_dashboard('start') #redis_port = ['6381','6382'] #定义所要启动 redis实例的端口号 redis_instance_port = {1:'6381',2:"6382"} for k,p in redis_instance_port.items(): #生成redis启动配置文件 template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%p,memsize=4) conf = open('./conf/%s.conf'%p,'w') conf.write(template_file) conf.close() #启动redis master 实例 codis_redis('start',p) print slot_range(1024,len(redis_instance_port.keys())) time.sleep(5) remove_fenc() slot_init() for k,v in slot_range(1024,len(redis_instance_port.keys())).items(): codis_group(k,'%s:%s'%(get_client_ip(),v[1]),'master') for k,v in slot_range(1024,len(redis_instance_port.keys())).items(): codis_initslot(k,v[0]) offline_proxy() codis_proxy('start') elif option == "task" and sys.argv[2] == "start_dashborad": codis_dashboard('start') elif option == "task" and sys.argv[2] == "stop_dashboard": codis_dashboard("stop") elif option == "task" and sys.argv[2] == "start_redis" and sys.argv[3] is not None: #redis_tmp_port = "6381" codis_redis('start',sys.argv[3]) elif option == "task" and sys.argv[2] == "stop_redis" and sys.argv[3] is not None: codis_redis('stop',sys.argv[3]) elif option == "task" and sys.argv[2] == "start_proxy": codis_proxy("start") elif option == "task" and sys.argv[2] == "stop_proxy": codis_proxy("stop") elif option == "task" and sys.argv[2] == "stopall": codis_proxy("stop") commands.getstatusoutput("killall codis-server") codis_dashboard("stop") else: help_prompt() #一般情况下codis集群在安装部署初始化的时候，需要先期规划，已经做好addgrop和initslot工作，后续看需求再补充 

2,twemproxy方案安装:

安装过程参考:

https://github.com/twitter/twemproxy

2.1注意，configure 编译之前需要系统安装autoreconf  
    当前用的 autoconf-2.64.tar.gz 这个包，
    tar zxvf autoconf-2.64.tar.gz ; cd autoconf-2.64 ; make && make install 即可
2.2 下载安装 twemproxy
    git clone git@github.com:twitter/twemproxy.git
    cd twemproxy
    autoreconf -fvi
    mkdir /usr/local/twemporxy
    ./configure –prefix=/usr/local/twemproxy --enable-debug=full
    make
    make install 
    src/nutcracker -h 

2.3 配置 twemproxy 代码 cat /usr/local/twemproxy/etc/544cluster.yml

    alpha:

    listen: 0.0.0.0:12000

    hash: fnv1a_64

    distribution: ketama

    auto_eject_hosts: true

    redis: true

    server_retry_timeout: 2000

    server_failure_limit: 1

    servers:

    - 192.168.5.44:6479:1

    - 192.168.5.16:6479:1

配置详解参考如下:

     http://cpjsjxy.iteye.com/blog/2090333

3.haproxy 安装部署：

3.1下载 haproxy-1.5.10.tar.gz tar zxvf haproxy-1.5.10.tar.gz ; cd haproxy-1.5.10; make TARGET=linux26 ARCH=x86_64&& make install 3.2 配置文件：

global

    daemon

    nbproc 6

    pidfile /var/run/haproxy.pid

    ulimit-n 65535



defaults

    mode tcp                        #mode { tcp|http|health }，tcp 表示4层，http表示7层，health仅作为健康检查使用

    retries 2                       #尝试2次失败则从集群摘除

    option redispatch               #如果失效则强制转换其他服务器

    option abortonclose             #连接数过大自动关闭

    maxconn 1024                    #最大连接数

    timeout connect 1d              #连接超时时间，重要，hive查询数据能返回结果的保证

    timeout client 1d               #同上

    timeout server 1d               #同上

    timeout check 2000              #健康检查时间

    log 127.0.0.1 local0 err #[err warning info debug]



listen  admin_stats                     #定义管理界面

    bind 0.0.0.0:8888               #管理界面访问IP和端口

    mode http                       #管理界面所使用的协议

    maxconn 10          #最大连接数

    stats refresh 30s               #30秒自动刷新

    stats uri /                     #访问url

    stats realm Hive\ Haproxy       #验证窗口提示

    stats auth admin:123456         #401验证用户名密码



listen codis-proxy-ha-20000        #codis-proxy

    bind 0.0.0.0:20000             #ha作为proxy所绑定的IP和端口

    mode tcp                        #以4层方式代理，重要

    balance leastconn               #调度算法 'leastconn' 最少连接数分配，或者 'roundrobin'，轮询分配

    maxconn 1024                    #最大连接数

    server codis-proxy2-master 192.168.5.15:19000  check inter 5000 rise 1 fall 2

    server codis-proxy1-master 192.168.5.14:19000  check inter 5000 rise 1 fall 2

    #server codis-server-5156381-master 192.168.5.15:6381  check inter 5000 rise 1 fall 2

    #server codis-server-5156382-master 192.168.5.15:6382  check inter 5000 rise 1 fall 2





listen twemproxy-ha-21000             

    bind 0.0.0.0:21000              

    mode tcp                        

    balance leastconn              

    maxconn 1024 

    server redis-544-master 192.168.5.44:6479  check inter 5000 rise 1 fall 2

    server redis-516-master 192.168.5.16:6479  check inter 5000 rise 1 fall 2



3.3 haproxy 启动关闭：

  /usr/local/sbin/haproxy -f /data/setup/haproxy-1.5.10/conf/haproxy.cfg    

4.keepalived安装部署: yum install keepalived

  主从配置是一样的，特别注意的配置项：
   state BACKUP
 为了避免网络不稳定的情况下，主从抢占，配为BACKUP
   interface em2
 VIP 邦定的网卡接口指定

cat /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
notification_email {
wangqiankun@lashou-inc.com
}
notification_email_from nagios@lashou.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ha_01
}

vrrp_instance HA_01 {
state BACKUP
nopreempt
interface em2
virtual_router_id 15
priority 99
advert_int 1
authentication {
    auth_type PASS
    auth_pass 2222
}
virtual_ipaddress {
    192.168.5.90
    192.168.5.91
}
}

virtual_server 192.168.5.90 20000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
#  persistence_timeout 10
protocol TCP

real_server 192.168.5.44 20000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 20000
    }
}

real_server 192.168.5.16 20000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 20000
    }
}

}
virtual_server 192.168.5.91 21000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
#   persistence_timeout 10
protocol TCP


real_server 192.168.5.44 21000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 21000
    }
}

real_server 192.168.5.16 21000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 21000
    }
}

}

proxy 做为keepalived + lvs 的后端真实机 需要安装lvs 并加如下脚本

    cat lvs_re.sh 
    #!/bin/bash
    WEB_VIP1=192.168.5.90
    WEB_VIP2=192.168.5.91

    . /etc/rc.d/init.d/functions

    case "$1" in
    start)
    ifconfig lo:0 $WEB_VIP1 netmask 255.255.255.255 broadcast $WEB_VIP1
    ifconfig lo:1 $WEB_VIP2 netmask 255.255.255.255 broadcast $WEB_VIP2
    /sbin/route add -host $WEB_VIP1 dev lo:0
    /sbin/route add -host $WEB_VIP2 dev lo:1
    echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
    sysctl -p >/dev/null 2>&1
    echo "RealServer Start OK"

    ;;
    stop)
    ifconfig lo:0 down
    ifconfig lo:1 down
    route del $WEB_VIP1 >/dev/null 2>&1
    route del $WEB_VIP2 >/dev/null 2>&1
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
    echo "RealServer Stoped"
    ;;

    status)
        # Status of LVS-DR real server.
        islothere=`/sbin/ifconfig lo:0 | grep "$WEB_VIP1"`
        islothere=`/sbin/ifconfig lo:1 | grep "$WEB_VIP2"`
        isrothere=`netstat -rn | grep "lo:0" | grep "$WEB_VIP"`
        if [ ! "$islothere" -o ! "isrothere" ];then
        # Either the route or the lo:0 device
        # not found.
        echo "LVS-DR real server Stopped."
        else
        echo "LVS-DR Running."
        fi
    ;;
    *)
        # Invalid entry.
        echo "$0: Usage: $0 {start|status|stop}"
        exit 1
    ;;
    esac
    exit 0

4.2.启动keepalived 和 lvs 服务加载配置

/etc/init.d/keepalived start ./lvs_re.sh start

六： 日常维护记录： 后续维护过程中待记录

转载于:https://www.cnblogs.com/shantu/p/4589798.html