1,go环境安装
[root@ansible-11 go]# tree
.
├── hosts
├── install_go.yaml
└── pkg
└── go1.7.1.linux-amd64.tar.gz
[root@ansible-11 go]# cat install_go.yaml
---
- hosts: go
vars:
remote_user: root
gather_facts: false
tasks:
- name: 分发go二进制包
unarchive: src=pkg/go1.7.1.linux-amd64.tar.gz dest=/usr/local/
- name: 添加到环境变量
shell: sed -i '$a export GOROOT=/usr/local/go' /etc/profile &&
sed -i '$a export GOPATH=/usr/data/gowork' /etc/profile &&
sed -i '$a export PATH=$PATH:$JAVA_HOME/bin:$GOROOT/bin:$GOPATH/bin:' /etc/profile &&
source /etc/profile
- name: 更新配置文件
shell: source /etc/profile
2,jdk 环境安装
[root@ansible-11 JDK]# cat install_java.yaml
---
- hosts: jdk
vars:
remote_user: root
gather_facts: false
tasks:
- name: 分发java二进制包
unarchive: src=pkg/jdk-8u191-linux-x64.tar.gz dest=/tmp/
- name: mv file
shell: mv /tmp/jdk1.8.0_191 /usr/local/jdk
- name: 添加到环境变量
shell: sed -i '$a export JAVA_HOME=/usr/local/jdk' /etc/profile &&
sed -i '$a export TOMCAT_HOME=/apps/tomcat' /etc/profile &&
sed -i '$a export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$TOMCAT_HOME/bin:$PATH' /etc/profile &&
sed -i '$a export CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar' /etc/profile &&
source /etc/profile
- name: 更新配置文件
shell: source /etc/profile
3,安装 godep
cd /usr/data/gowork/src/github.com/tools/
tar -xf godep-74.tar.gz
mv godep-74 godep
cd godep
go install ./
ls /usr/data/gowork/bin/
4,codis-redis安装
mkdir /usr/data/gowork/src/github.com/CodisLabs -p
cd /usr/data/gowork/src/github.com/CodisLabs
tar -xf codis-3.10-release.tar.gz
cd codis
make
mkdir -p /usr/local/codis/{logs,conf}
cp -r /usr/data/gowork/src/github.com/CodisLabs/codis/bin/ /usr/local/codis/
6,codis-server启动redis
mkdir -p /usr/local/codis/conf/redis_conf/
cp /usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/redis.conf /usr/local/codis/conf/redis_conf/redis-6379.conf
vim /usr/local/codis/conf/redis_conf/redis-6379.conf
# bind 127.0.0.1
port 6379
daemonize yes
requirepass mldnjava
pidfile /usr/data/redis/redis-6379/run/redis_6379.pid
logfile /usr/data/redis/redis-6379/logs/logs_6379.log
dir /usr/data/redis/redis-6379/db
timeout 60
maxclients 60000 #最大连接数
appendonly yes #开启aof 存储
appendfilename "appendonly.aof"
cp /usr/local/codis/conf/redis_conf/redis-6379.conf /usr/local/codis/conf/redis_conf/redis-6380.conf
cp /usr/local/codis/conf/redis_conf/redis-6379.conf /usr/local/codis/conf/redis_conf/redis-6381.conf
sed -i 's#6379#6380#g' /usr/local/codis/conf/redis_conf/redis-6380.conf
sed -i 's#6379#6381#g' /usr/local/codis/conf/redis_conf/redis-6381.conf
mkdir -p /usr/data/redis/redis-6379/{db,logs,run}
mkdir -p /usr/data/redis/redis-6380/{db,logs,run}
mkdir -p /usr/data/redis/redis-6381/{db,logs,run}
echo "vm.overcommit_memory=1" >> /etc/sysctl.conf
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6379.conf
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6380.conf
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6381.conf
ss -nutlp | grep 63
5,zookeeper 集群安装
root@ansible-11 zookeeper]# ls
conf hosts install_zookeeper.sh pkg zoo1.yaml zoo2.yaml zoo3.yaml zoo_class.yaml zookeeper.yaml
[root@ansible-11 zookeeper]# tree
.
├── conf
│ └── zoo.cfg
├── hosts
├── install_zookeeper.sh
├── pkg
│ └── zookeeper-3.4.14.tar.gz
├── zoo1.yaml
├── zoo2.yaml
├── zoo3.yaml
├── zoo_class.yaml
└── zookeeper.yaml
6,生成dashboard配置文件
/usr/local/codis/bin/codis-dashboard --default-config | tee /usr/local/codis/conf/dashboard.ini
修改配置文件
[root@localhost ~]# cat /usr/local/codis/conf/dashboard.ini
##################################################
# #
# Codis-Dashboard #
# #
##################################################
# Set Coordinator, only accept "zookeeper" & "etcd".
coordinator_name = "zookeeper" #调试者名称zookeeper 或者etcd
coordinator_addr = "10.1.234.127:2181,10.1.234.128:2181,10.1.234.129:2181"
# Set Codis Product Name/Auth.
product_name = "mldn-codis" #key 名称
product_auth = "mldnjava" #密码
# Set bind address for admin(rpc), tcp only.
admin_addr = "0.0.0.0:18080" #访问地址
# Set quorum value for sentinel, default is 2.
sentinel_quorum = 2
启动服务
[root@localhost ~]# nohup /usr/local/codis/bin/codis-dashboard --ncpu=2 --config=/usr/local/codis/conf/dashboard.ini --log=/usr/local/codis/logs/dashboard.log --log-level=WARN > /dev/null 2>& 1 &
7,生成FE 配置文件
/usr/local/codis/bin/codis-admin --dashboard-list --zookeeper=10.1.234.127,10.1.234.128,10.1.234.129 | tee /usr/local/codis/conf/codis.json
[root@localhost ~]# cat /usr/local/codis/conf/codis.json
[
{
"name": "mldn-codis",
"dashboard": "10.1.234.127:18080"
}
]
启动FE
nohup /usr/local/codis/bin/codis-fe --ncpu=2 --log=/usr/local/codis/logs/fe.log --log-level=WARN --dashboard-list=/usr/local/codis/conf/codis.json --listen=0.0.0.0:18090 > /dev/null 2>& 1 &
8,配置proxy 代理
生成配置文件
/usr/local/codis/bin/codis-proxy --default-config | tee /usr/local/codis/conf/proxy.ini
修改配置文件
vim /usr/local/codis/conf/proxy.ini
product_name = "mldn-codis"
product_auth = "mldnjava"
admin_addr = "0.0.0.0:11080"
jodis_addr = "10.1.234.127:2181,10.1.234.128:2181,10.1.234.129:2181"
启动服务
nohup /usr/local/codis/bin/codis-proxy --ncpu=2 --config=/usr/local/codis/conf/proxy.ini --log=/usr/local/codis/logs/proxy.log --log-level=WARN > /dev/null 2>& 1 &
9,行创建组,也可以用图形界面FE
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=1
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=2
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-group --gid=3
添加redis到组,这里添加没有主从的关系
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=1 --addr=10.1.234.127:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=2 --addr=10.1.234.128:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --group-add --gid=3 --addr=10.1.234.129:6381
插槽的分配
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=1 --end=300 --gid=1
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=301 --end=700 --gid=2
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --slot-action --create-range --beg=701 --end=1023 --gid=3
添加代理
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.127:11080
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.128:11080
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --create-proxy --addr=10.1.234.129:11080
设置主从关系
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.127:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.128:6381
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6379
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6380
/usr/local/codis/bin/codis-admin --dashboard=10.1.234.127:18080 --sync-action --create --addr=10.1.234.129:6381
10,测试压测,zookeeper使用
测试
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-cli -h 10.1.234.127 -a mldnjava -p 19000
10.1.234.127:19000> set x 100
OK
10.1.234.127:19000> KEYS *
Error: Server closed the connection
10.1.234.127:19000> get x
"100"
压测
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-benchmark -h 10.1.234.127 -p 19000 -c 1000 -n 100 -d 10000
进入zookeerper
/usr/local/zookeeper/bin/zkCli.sh -server 10.1.234.127
查看
ls /
删除
rmr /codis3
11,负载均衡haproxy
tar xzvf haproxy-1.5.18.tar.gz -C /usr/local/src/
cd /usr/local/src/haproxy-1.5.18/
make TARGET=linux26 PREFIX=/usr/local/haproxy ARCH=x86_64
make install PREFIX=/usr/local/haproxy
mkdir -p /usr/data/haproxy
cp /usr/local/src/haproxy-1.5.18/examples/haproxy.cfg /usr/local/haproxy/
[root@localhost haproxy]# cat /usr/local/haproxy/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
#log loghost local0 info
maxconn 4096
chroot /usr/local/haproxy
pidfile /usr/data/haproxy/haproxy.pid
uid 99
gid 99
daemon
#debug
#quiet
defaults
log global
mode tcp
option abortonclose
option redispatch
retries 3
maxconn 2000
timeout connect 5000
timeout client 50000
timeout server 50000
listen proxy_status
bind :16001
mode tcp
balance roundrobin
server codis_proxy_1 10.1.234.127:19000 check inter 10s
server codis_proxy_2 10.1.234.128:19000 check inter 10s
server codis_proxy_3 10.1.234.129:19000 check inter 10s
frontend admin_stats
bind :7777
mode http
stats enable
option httplog
maxconn 10
stats refresh 30s
stats uri /admin
stats auth mldn:java
stats hide-version
stats admin if TRUE
启动服务
[root@localhost haproxy]# /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg
通过代理来测试
/usr/data/gowork/src/github.com/CodisLabs/codis/extern/redis-3.2.4/src/redis-benchmark -h 10.1.234.127 -p 16001
12, vip keepalive
yum -y install keepalived
[root@localhost ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lvs_master1
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 172
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.234.130
}
}
virtual_server 10.1.234.130 16001 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
real_server 10.1.234.127 16001 {
weight 3
notify_down /opt/shells/keepalived_mysql.sh
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 16001
}
}
}
[root@localhost ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lvs_master1
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 172
priority 99
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.234.130
}
}
virtual_server 10.1.234.130 16001 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
real_server 10.1.234.128 16001 {
weight 3
notify_down /opt/shells/keepalived_mysql.sh
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 16001
}
}
}
[root@localhost haproxy-1.5.18]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lvs_master1
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 172
priority 98
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.234.130
}
}
virtual_server 10.1.234.130 16001 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
real_server 10.1.234.129 16001 {
weight 3
notify_down /opt/shells/keepalived_mysql.sh
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 16001
}
}
}
systemctl restart keepalived
systemctl enable keepalived
13,守护进程安装
yum -y install supervisord
systemctl restart supervisord
systemctl enable supervisord
[root@localhost ~]# cat /etc/supervisord.d/codis-redis.ini
[program:codis]
command=sh /win/sh/codis-redis.sh
directory=/win/sh/
autostart=true
autorestart=true
logfile_maxbytes=50MB
logfile_backups=10
loglevel=info
stderr_logfile=/usr/local/codis/logs/codis.err.log
stdout_logfile=/usr/local/codis/logs/codis.out.log
user=root
stopsignal=INT
redirect_stderr=true
startsecs=10
[root@localhost ~]# cat /win/sh/codis-redis.sh
#!/bin/bash
while :
do
port1=`ss -nutlp | grep 6379 |wc -l`
port2=`ss -nutlp | grep 6380 |wc -l`
port3=`ss -nutlp | grep 6381 |wc -l`
port4=`ss -nutlp | grep 18080 |wc -l`
port5=`ss -nutlp | grep 18090 |wc -l`
port6=`ss -nutlp | grep 11080 |wc -l`
port7=`ss -nutlp | grep 7777 |wc -l`
port8=`ss -nutlp | grep 2181 |wc -l`
if [ $port1 -lt 1 ];then
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6379.conf
fi
if [ $port2 -lt 1 ];then
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6380.conf
fi
if [ $port3 -lt 1 ];then
/usr/local/codis/bin/codis-server /usr/local/codis/conf/redis_conf/redis-6381.conf
fi
#Codis-Dashboard
if [ $port4 -lt 1 ];then
nohup /usr/local/codis/bin/codis-dashboard --ncpu=2 --config=/usr/local/codis/conf/dashboard.ini --log=/usr/local/codis/logs/dashboard.log --log-level=WARN > /dev/null 2>& 1 &
fi
#Codis-FE
if [ $port5 -lt 1 ];then
nohup /usr/local/codis/bin/codis-fe --ncpu=2 --log=/usr/local/codis/logs/fe.log --log-level=WARN --dashboard-list=/usr/local/codis/conf/codis.json --listen=0.0.0.0:18090 > /dev/null 2>& 1 &
fi
#Codis-proxy
if [ $port6 -lt 1 ];then
nohup /usr/local/codis/bin/codis-proxy --ncpu=2 --config=/usr/local/codis/conf/proxy.ini --log=/usr/local/codis/logs/proxy.log --log-level=WARN > /dev/null 2>& 1 &
fi
#HA-proxy
if [ $port7 -lt 1 ];then
/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg
fi
#zookp
if [ $port8 -lt 1 ];then
/usr/local/zookeeper/bin/zkServer.sh start
fi
done
systemctl restart supervisord