Centos7 源代码安装Redis Cluster 6.2.9 Redis Cluster Proxy

安装与编译

sudo su -

yum install -y gcc g++ gcc-c++ make tcl

cd /tmp && wget http://download.redis.io/releases/redis-6.2.9.tar.gz

tar -zxvf redis-6.2.9.tar.gz && cd redis-6.2.9

make -j8 # 利用多核心编译否则时间会比较长 ,若是第一次在这边配置上编译建议先make test

make PREFIX=/usr/local install # 自定义安装路径

cat /tmp/redis-6.2.9/redis.conf |grep -v '^#' |grep -v "^$" > /etc/redis/redis-6379.conf #去掉官方配置文件里面的注释和空行

系统调优

echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf

echo 2048 > /proc/sys/net/core/somaxconn

echo 'net.core.somaxconn=2048' >> /etc/sysctl.conf && sysctl -p

sysctl vm.overcommit_memory=1

开启集群和限制单个节点的内存以及淘汰策略

echo 'maxmemory 64GB

maxmemory-policy volatile-ttl

cluster-enabled yes

cluster-node-timeout 10000' >>/etc/redis/redis-6379.conf

cat /etc/redis/redis-6379.conf

生成多端口的配置文件

#服务器有512G内存,单实例分配了64G,所以启动8个实例(压力大可以少开一个)

#复制文件

for i in {8001..8008} ;do cp /etc/redis/redis-6379.conf "/etc/redis/redis-${i}.conf" ;done

#替换文件里面的端口

for i in {8001..8008};do

config_file="/etc/redis/redis-${i}.conf"

sed -i "s/6379/${i}/g" $config_file

done

分发配置文件并创建文件夹

ansible redis -m copy -a "src=/etc/redis dest=/etc/"

ansible redis -m shell -a "mkdir /data/log/"

ansible redis -m shell -a "mkdir -p /data/redis/{8001..8008}"

启动与测试

启动单个节点,完成以后需要手动停止

redis-server /etc/redis/redis-6379.conf #

#启动服务,可以做成shell,用ansible启动

for i in {8001..8008} ; do

redis-server "/etc/redis/redis-${i}.conf" ;

done

bind 0.0.0.0
protected-mode yes
port 8001
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
pidfile /var/run/redis_8001.pid
loglevel notice
logfile "/data/log/redis_8001.log"
databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /data/redis/8001
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events "KEA"
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes
cluster-node-timeout 10000
maxmemory 32GB
maxmemory-policy volatile-ttl
cluster-enabled yes

# 遇到占用空间很大的问题增加下面的配置,自动碎片整理

activedefrag yes
# Minimum amount of fragmentation waste to start active defrag
# 内存碎片达到多少的时候开启整理
active-defrag-ignore-bytes 100mb
# Minimum percentage of fragmentation to start active defrag
# 碎片率达到百分之多少开启整理
active-defrag-threshold-lower 10

若是不方便重启可以直接连接到对应的节点执行下面的命令

开启自动内存碎片整理 config set activedefrag yes

手动清理内存碎片 memory purge

# 关停服务,非必须

#pkill redis

构建集群 64G*6*8

redis-cli --cluster create --cluster-replicas 1 172.20.192.22:8001 172.20.192.22:8002 172.20.192.22:8003 172.20.192.22:8004 172.20.192.23:8001 172.20.192.23:8002 172.20.192.23:8003 172.20.192.23:8004 172.20.192.31:8001 172.20.192.31:8002 172.20.192.31:8003 172.20.192.31:8004 172.20.192.32:8001 172.20.192.32:8002 172.20.192.32:8003 172.20.192.32:8004 172.20.192.33:8001 172.20.192.33:8002 172.20.192.33:8003 172.20.192.33:8004 172.20.192.34:8001 172.20.192.34:8002 172.20.192.34:8003 172.20.192.34:8004 172.20.192.22:8005 172.20.192.22:8006 172.20.192.22:8007 172.20.192.22:8008 172.20.192.23:8005 172.20.192.23:8006 172.20.192.23:8007 172.20.192.23:8008 172.20.192.31:8005 172.20.192.31:8006 172.20.192.31:8007 172.20.192.31:8008 172.20.192.32:8005 172.20.192.32:8006 172.20.192.32:8007 172.20.192.32:8008 172.20.192.33:8005 172.20.192.33:8006 172.20.192.33:8007 172.20.192.33:8008 172.20.192.34:8005 172.20.192.34:8006 172.20.192.34:8007 172.20.192.34:8008

Redis cluster proxy 解决应用连接节点太多的

2023年7月25更新:官方的版本一直没有更新,大量使用后有奔溃问题,可以考虑使用.
https://github.com/domrod/redis-cluster-proxy/archive/refs/tags/1.0.3.tar.gz

GitHub - RedisLabs/redis-cluster-proxy: A proxy for Redis clusters.

附:完整的配置文件

cd /tmp && git clone GitHub - RedisLabs/redis-cluster-proxy: A proxy for Redis clusters.

# 只是临时用一下,退出后失效

yum install centos-release-scl

yum install devtoolset-7-gcc*

scl enable devtoolset-7 bash # source /opt/rh/devtoolset-7/enable

gcc -v

make

make PREFIX=/usr/local install

配置文件:

bind 0.0.0.0
port 16379
connections-pool-size 10
connections-pool-min-size 10
daemonize yes
enable-cross-slot yes
log-level error
logfile "/data/log/proxy.log"
max-clients 5000
pidfile /var/run/redis-cluster-proxy.pid
threads 16
entry-point 172.20.192.22:8001
entry-point 172.20.192.22:8002
entry-point 172.20.192.22:8003
entry-point 172.20.192.22:8004
entry-point 172.20.192.23:8001
entry-point 172.20.192.23:8002
entry-point 172.20.192.23:8003
entry-point 172.20.192.23:8004
entry-point 172.20.192.31:8001
entry-point 172.20.192.31:8002
entry-point 172.20.192.31:8003
entry-point 172.20.192.31:8004
entry-point 172.20.192.32:8001
entry-point 172.20.192.32:8002
entry-point 172.20.192.32:8003
entry-point 172.20.192.32:8004
entry-point 172.20.192.33:8001
entry-point 172.20.192.33:8002
entry-point 172.20.192.33:8003
entry-point 172.20.192.33:8004
entry-point 172.20.192.34:8001
entry-point 172.20.192.34:8002
entry-point 172.20.192.34:8003
entry-point 172.20.192.34:8004
entry-point 172.20.192.22:8005
entry-point 172.20.192.22:8006
entry-point 172.20.192.22:8007
entry-point 172.20.192.22:8008
entry-point 172.20.192.23:8005
entry-point 172.20.192.23:8006
entry-point 172.20.192.23:8007
entry-point 172.20.192.23:8008
entry-point 172.20.192.31:8005
entry-point 172.20.192.31:8006
entry-point 172.20.192.31:8007
entry-point 172.20.192.31:8008
entry-point 172.20.192.32:8005
entry-point 172.20.192.32:8006
entry-point 172.20.192.32:8007
entry-point 172.20.192.32:8008
entry-point 172.20.192.33:8005
entry-point 172.20.192.33:8006
entry-point 172.20.192.33:8007
entry-point 172.20.192.33:8008
entry-point 172.20.192.34:8005
entry-point 172.20.192.34:8006
entry-point 172.20.192.34:8007
entry-point 172.20.192.34:8008

# 启动

redis-cluster-proxy -c /etc/redis/proxy.conf

为了保障测试需要安装ruby 以2.7为例

gpg2 --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
curl -sSL https://get.rvm.io | bash -s stable

source /etc/profile.d/rvm.sh
rvm install 2.7
ruby -v
gem install bundler:1.16.6
# --bundle update --bundle
make test

维护 减少内存占用

for i in {8001..8008};do

config_file="/etc/redis/redis-${i}.conf"

sed -i "s/64GB/32GB/g" $config_file

done

# 关停应用,防止出现进程不存在

ps -ef | grep 8008 |grep -v grep |awk '{print $2}' | xargs kill

# 启动应用
redis-server /etc/redis/redis-8008.conf

# 查看进程
  ps -ef |grep redis