codis 搭建高可用集群

1,go环境安装
 

[root@ansible-11 go]# tree
.
├── hosts
├── install_go.yaml
└── pkg
    └── go1.7.1.linux-amd64.tar.gz

[root@ansible-11 go]# cat install_go.yaml 
---
- hosts: go
  vars: 
  remote_user: root
  gather_facts: false
  
  tasks:
  - name: 分发go二进制包 
    unarchive: src=pkg/go1.7.1.linux-amd64.tar.gz dest=/usr/local/

  - name: 添加到环境变量
    shell: sed -i '$a export GOROOT=/usr/local/go' /etc/profile &&
           sed -i '$a export GOPATH=/usr/data/gowork' /etc/profile && 
           sed -i '$a export PATH=$PATH:$JAVA_HOME/bin:$GOROOT/bin:$GOPATH/bin:' /etc/profile && 
           source /etc/profile
  
  - name: 更新配置文件
    shell: source /etc/profile

2,jdk 环境安装

[root@ansible-11 JDK]# cat install_java.yaml 
---
- hosts: jdk
  vars: 
  remote_user: root
  gather_facts: false
  
  tasks:
  - name: 分发java二进制包 
    unarchive: src=pkg/jdk-8u191-linux-x64.tar.gz  dest=/tmp/

  - name: mv file
    shell: mv /tmp/jdk1.8.0_191 /usr/local/jdk 
  - name: 添加到环境变量
    shell: sed -i '$a export JAVA_HOME=/usr/local/jdk' /etc/profile &&
           sed -i '$a export TOMCAT_HOME=/apps/tomcat' /etc/profile && 
           sed -i '$a export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH' /etc/profile && 
           sed -i '$a export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar' /etc/profile && 
           source /etc/profile
  
  - name: 更新配置文件
    shell: source /etc/profile

3,安装 godep
 

cd /usr/data/gowork/src/github.com/tools/
tar -xf godep-74.tar.gz
mv godep-74 godep
cd godep
go install ./
ls /usr/data/gowork/bin/ 

4,codis-redis安装

mkdir /usr/data/gowork/src/github.com/CodisLabs -p
cd /usr/data/gowork/src/github.com/CodisLabs
tar -xf codis-3.10-release.tar.gz
cd codis
make 

mkdir -p /usr/local/codis/{logs,conf}
cp -r /usr/data/gowork/src/github.com/CodisLabs/codis/bin/ /usr/local/codis/

6,codis-server启动redis

mkdir -p /usr/local/codis/conf/redis_conf/

cp /usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/redis.conf /usr/local/codis/conf/redis_conf/redis-6379.conf


vim /usr/local/codis/conf/redis_conf/redis-6379.conf
# bind 127.0.0.1
port 6379
daemonize yes
requirepass mldnjava
pidfile /usr/data/redis/redis-6379/run/redis_6379.pid
logfile /usr/data/redis/redis-6379/logs/logs_6379.log
dir /usr/data/redis/redis-6379/db
timeout 60
maxclients 60000            #最大连接数
appendonly yes            #开启aof 存储
appendfilename "appendonly.aof"


cp /usr/local/codis/conf/redis_conf/redis-6379.conf /usr/local/codis/conf/redis_conf/redis-6380.conf
cp /usr/local/codis/conf/redis_conf/redis-6379.conf /usr/local/codis/conf/redis_conf/redis-6381.conf
sed -i 's#6379#6380#g' /usr/local/codis/conf/redis_conf/redis-6380.conf
sed -i 's#6379#6381#g' /usr/local/codis/conf/redis_conf/redis-6381.conf

mkdir -p /usr/data/redis/redis-6379/{db,logs,run}
mkdir -p /usr/data/redis/redis-6380/{db,logs,run}
mkdir -p /usr/data/redis/redis-6381/{db,logs,run}


echo "vm.overcommit_memory=1" >> /etc/sysctl.conf


/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6379.conf
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6380.conf
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6381.conf


ss -nutlp | grep 63

5,zookeeper 集群安装

root@ansible-11 zookeeper]# ls
conf  hosts  install_zookeeper.sh  pkg  zoo1.yaml  zoo2.yaml  zoo3.yaml  zoo_class.yaml  zookeeper.yaml
[root@ansible-11 zookeeper]# tree 
.
├── conf
│   └── zoo.cfg
├── hosts
├── install_zookeeper.sh
├── pkg
│   └── zookeeper-3.4.14.tar.gz
├── zoo1.yaml
├── zoo2.yaml
├── zoo3.yaml
├── zoo_class.yaml
└── zookeeper.yaml

 6,生成dashboard配置文件

/usr/local/codis/bin/codis-dashboard  --default-config | tee /usr/local/codis/conf/dashboard.ini

修改配置文件
[root@localhost ~]# cat  /usr/local/codis/conf/dashboard.ini

##################################################
#                                                #
#                  Codis-Dashboard               #
#                                                #
##################################################

# Set Coordinator, only accept "zookeeper" & "etcd".
coordinator_name = "zookeeper"  #调试者名称zookeeper 或者etcd
coordinator_addr = "10.1.234.127:2181,10.1.234.128:2181,10.1.234.129:2181"

# Set Codis Product Name/Auth.
product_name = "mldn-codis"    #key 名称
product_auth = "mldnjava"     #密码

# Set bind address for admin(rpc), tcp only.
admin_addr = "0.0.0.0:18080"    #访问地址

# Set quorum value for sentinel, default is 2.
sentinel_quorum = 2
启动服务
[root@localhost ~]# nohup /usr/local/codis/bin/codis-dashboard --ncpu=2 --config=/usr/local/codis/conf/dashboard.ini --log=/usr/local/codis/logs/dashboard.log --log-level=WARN  > /dev/null 2>& 1 &

7,生成FE 配置文件

/usr/local/codis/bin/codis-admin --dashboard-list --zookeeper=10.1.234.127,10.1.234.128,10.1.234.129 | tee /usr/local/codis/conf/codis.json

[root@localhost ~]# cat /usr/local/codis/conf/codis.json
[
    {
        "name": "mldn-codis",
        "dashboard": "10.1.234.127:18080"
    }
]
启动FE
nohup /usr/local/codis/bin/codis-fe --ncpu=2 --log=/usr/local/codis/logs/fe.log --log-level=WARN --dashboard-list=/usr/local/codis/conf/codis.json --listen=0.0.0.0:18090 > /dev/null 2>& 1 &

8,配置proxy 代理

生成配置文件
/usr/local/codis/bin/codis-proxy --default-config | tee /usr/local/codis/conf/proxy.ini
修改配置文件
vim /usr/local/codis/conf/proxy.ini

product_name = "mldn-codis"
product_auth = "mldnjava"
admin_addr = "0.0.0.0:11080"
jodis_addr = "10.1.234.127:2181,10.1.234.128:2181,10.1.234.129:2181"


启动服务
nohup /usr/local/codis/bin/codis-proxy --ncpu=2 --config=/usr/local/codis/conf/proxy.ini --log=/usr/local/codis/logs/proxy.log --log-level=WARN  > /dev/null 2>& 1 &

9,行创建组,也可以用图形界面FE

/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=1
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=2
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=3


添加redis到组,这里添加没有主从的关系
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6381



插槽的分配
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=1 --end=300 --gid=1
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=301 --end=700 --gid=2
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=701 --end=1023 --gid=3

添加代理
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.127:11080
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.128:11080
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.129:11080


设置主从关系
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6381

10,测试压测,zookeeper使用

测试
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-cli -h 10.1.234.127 -a mldnjava -p 19000
10.1.234.127:19000> set x 100
OK
10.1.234.127:19000> KEYS *
Error: Server closed the connection
10.1.234.127:19000> get x
"100"


压测
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-benchmark -h 10.1.234.127 -p 19000 -c 1000 -n 100 -d 10000

进入zookeerper 
/usr/local/zookeeper/bin/zkCli.sh -server 10.1.234.127
查看
ls /
删除 
rmr /codis3

11,负载均衡haproxy

tar xzvf haproxy-1.5.18.tar.gz -C /usr/local/src/
cd /usr/local/src/haproxy-1.5.18/
make TARGET=linux26 PREFIX=/usr/local/haproxy ARCH=x86_64
make install PREFIX=/usr/local/haproxy
mkdir -p /usr/data/haproxy
cp /usr/local/src/haproxy-1.5.18/examples/haproxy.cfg /usr/local/haproxy/

[root@localhost haproxy]# cat   /usr/local/haproxy/haproxy.cfg
global
        log 127.0.0.1   local0
        log 127.0.0.1   local1 notice
        #log loghost    local0 info
        maxconn 4096
        chroot /usr/local/haproxy
        pidfile /usr/data/haproxy/haproxy.pid
        uid 99
        gid 99
        daemon
        #debug
        #quiet

defaults
        log     global
        mode    tcp
        option  abortonclose
        option redispatch
        retries 3
        maxconn 2000
        timeout connect 5000
        timeout client  50000
        timeout server  50000


listen proxy_status 
	bind :16001
        mode tcp
        balance roundrobin
        server codis_proxy_1 10.1.234.127:19000 check inter 10s
        server codis_proxy_2 10.1.234.128:19000 check inter 10s
        server codis_proxy_3 10.1.234.129:19000 check inter 10s

frontend admin_stats 
	bind :7777
        mode http
        stats enable
        option httplog
        maxconn 10
        stats refresh 30s
        stats uri /admin
        stats auth mldn:java
        stats hide-version
        stats admin if TRUE


启动服务
[root@localhost haproxy]# /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg


通过代理来测试
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-benchmark -h 10.1.234.127 -p 16001

12, vip keepalive

yum -y install keepalived

[root@localhost ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id lvs_master1
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 172
    priority 100
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.1.234.130
    }
}

virtual_server 10.1.234.130 16001 {
    delay_loop 6
    lb_algo rr
    lb_kind NAT
    nat_mask 255.255.255.0
    persistence_timeout 50
    protocol TCP

    real_server 10.1.234.127 16001 {
        weight 3
        notify_down /opt/shells/keepalived_mysql.sh
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 16001
       }
    }
}


[root@localhost ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id lvs_master1
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 172
    priority 99
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.1.234.130
    }
}

virtual_server 10.1.234.130 16001 {
    delay_loop 6
    lb_algo rr
    lb_kind NAT
    nat_mask 255.255.255.0
    persistence_timeout 50
    protocol TCP

    real_server 10.1.234.128 16001 {
        weight 3
        notify_down /opt/shells/keepalived_mysql.sh
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 16001
       }
    }
}


[root@localhost haproxy-1.5.18]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id lvs_master1
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 172
    priority 98
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.1.234.130
    }
}

virtual_server 10.1.234.130 16001 {
    delay_loop 6
    lb_algo rr
    lb_kind NAT
    nat_mask 255.255.255.0
    persistence_timeout 50
    protocol TCP

    real_server 10.1.234.129 16001 {
        weight 3
        notify_down /opt/shells/keepalived_mysql.sh
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 16001
       }
    }
}

systemctl restart keepalived
systemctl enable keepalived

13,守护进程安装

yum -y install supervisord
systemctl restart supervisord
systemctl enable supervisord

[root@localhost ~]#  cat /etc/supervisord.d/codis-redis.ini
[program:codis]
command=sh /win/sh/codis-redis.sh
directory=/win/sh/
autostart=true
autorestart=true
logfile_maxbytes=50MB
logfile_backups=10
loglevel=info
stderr_logfile=/usr/local/codis/logs/codis.err.log
stdout_logfile=/usr/local/codis/logs/codis.out.log
user=root
stopsignal=INT
redirect_stderr=true
startsecs=10

[root@localhost ~]# cat /win/sh/codis-redis.sh 
#!/bin/bash
while :
do
port1=`ss -nutlp | grep 6379 |wc -l`
port2=`ss -nutlp | grep 6380 |wc -l`
port3=`ss -nutlp | grep 6381 |wc -l`
port4=`ss -nutlp | grep 18080 |wc -l`
port5=`ss -nutlp | grep 18090 |wc -l`
port6=`ss -nutlp | grep 11080 |wc -l`
port7=`ss -nutlp | grep 7777 |wc -l`
port8=`ss -nutlp | grep 2181 |wc -l`

if [ $port1 -lt 1 ];then
  /usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6379.conf
fi
if [ $port2 -lt 1 ];then
  /usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6380.conf
fi
if [ $port3 -lt 1 ];then
  /usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6381.conf
fi
#Codis-Dashboard 
if [ $port4 -lt 1 ];then
  nohup /usr/local/codis/bin/codis-dashboard --ncpu=2 --config=/usr/local/codis/conf/dashboard.ini --log=/usr/local/codis/logs/dashboard.log --log-level=WARN  > /dev/null 2>& 1 &
fi
#Codis-FE
if [ $port5 -lt 1 ];then
  nohup /usr/local/codis/bin/codis-fe --ncpu=2 --log=/usr/local/codis/logs/fe.log --log-level=WARN --dashboard-list=/usr/local/codis/conf/codis.json --listen=0.0.0.0:18090 > /dev/null 2>& 1 &
fi
#Codis-proxy
if [ $port6 -lt 1 ];then
  nohup /usr/local/codis/bin/codis-proxy --ncpu=2 --config=/usr/local/codis/conf/proxy.ini --log=/usr/local/codis/logs/proxy.log --log-level=WARN  > /dev/null 2>& 1 &
fi
#HA-proxy
if [ $port7 -lt 1 ];then
  /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg
fi
#zookp
if [ $port8 -lt 1 ];then
  /usr/local/zookeeper/bin/zkServer.sh start
fi
done


systemctl restart supervisord

 

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值