Helm部署高可用Redis Sentinel

1. 下载

[root@master helm]# helm search stable/redis
NAME           	CHART VERSION	APP VERSION	DESCRIPTION                                                 
stable/redis   	10.1.0       	5.0.7      	Open source, advanced key-value store. It is often referr...
stable/redis-ha	4.1.1        	5.0.5      	Highly available Kubernetes implementation of Redis         
[root@master helm]# helm fetch stable/redis-ha
[root@master helm]# ls
redis-ha-4.1.1.tgz
# 解压
[root@master helm]# tar -xvf redis-ha-4.1.1.tgz
[root@master helm]# cd redis-ha/
[root@master redis-ha]# ls
Chart.yaml  ci  OWNERS  README.md  templates  values.yaml

2. 创建StorageClass

[root@master redis-ha]# cat <<EOF> redis-data-sc.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: redis-data
provisioner: fuseim.pri/ifs
EOF

[root@master redis-ha]# kubectl create -f redis-data-sc.yaml

3. 新建配置文件

[root@master redis-ha]# cat <<EOF> my-values.yaml
haproxy:
  enabled: true
  service:
    type: NodePort

persistentVolume:
  storageClass: redis-data
  size: "2Gi"

auth: true
redisPassword: www19930327
EOF

4. 安装

[root@master redis-ha]# helm install --name redis-ha -f my-values.yaml . --namespace kube-ops
Release "redis-ha" has been upgraded.
LAST DEPLOYED: Sun Dec  1 23:29:00 2019
NAMESPACE: kube-ops
STATUS: DEPLOYED

RESOURCES:
==> v1/ConfigMap
NAME                DATA  AGE
redis-ha-configmap  5     4m35s

==> v1/Deployment
NAME              READY  UP-TO-DATE  AVAILABLE  AGE
redis-ha-haproxy  0/3    3           0          0s

==> v1/Pod(related)
NAME                              READY  STATUS    RESTARTS  AGE
redis-ha-haproxy-8ddcdbfc8-hvgvb  0/1    Init:0/1  0         0s
redis-ha-haproxy-8ddcdbfc8-nbbz6  0/1    Init:0/1  0         0s
redis-ha-haproxy-8ddcdbfc8-x2srt  0/1    Init:0/1  0         0s
redis-ha-server-0                 2/2    Running   0         4m35s
redis-ha-server-1                 2/2    Running   0         4m22s
redis-ha-server-2                 2/2    Running   0         4m10s

==> v1/Role
NAME      AGE
redis-ha  4m35s

==> v1/RoleBinding
NAME      AGE
redis-ha  4m35s

==> v1/Service
NAME                 TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)             AGE
redis-ha             ClusterIP  None           <none>       6379/TCP,26379/TCP  4m35s
redis-ha-announce-0  ClusterIP  10.99.205.103  <none>       6379/TCP,26379/TCP  4m35s
redis-ha-announce-1  ClusterIP  10.106.77.208  <none>       6379/TCP,26379/TCP  4m35s
redis-ha-announce-2  ClusterIP  10.103.17.152  <none>       6379/TCP,26379/TCP  4m35s
redis-ha-haproxy     NodePort   10.97.173.27   <none>       6379:31995/TCP      0s

==> v1/ServiceAccount
NAME              SECRETS  AGE
redis-ha          1        4m35s
redis-ha-haproxy  1        0s

==> v1/StatefulSet
NAME             READY  AGE
redis-ha-server  3/3    4m35s


NOTES:
Redis can be accessed via port 6379 and Sentinel can be accessed via port 26379 on the following DNS name from within your cluster:
redis-ha.kube-ops.svc.cluster.local

To connect to your Redis server:
1. Run a Redis pod that you can use as a client:

   kubectl exec -it redis-ha-server-0 sh -n kube-ops

2. Connect using the Redis CLI:

  redis-cli -h redis-ha.kube-ops.svc.cluster.local

[root@master redis-ha]# kubectl get pods -n kube-ops | grep redis
redis-ha-haproxy-8ddcdbfc8-hvgvb               1/1     Running   0          13m
redis-ha-haproxy-8ddcdbfc8-nbbz6               1/1     Running   0          13m
redis-ha-haproxy-8ddcdbfc8-x2srt               1/1     Running   0          13m
redis-ha-server-0                              2/2     Running   0          18m
redis-ha-server-1                              2/2     Running   0          18m
redis-ha-server-2                              2/2     Running   0          18m

[root@master redis-ha]# kubectl get svc -n kube-ops | grep redis
harbor-harbor-redis           ClusterIP   10.111.20.67     <none>        6379/TCP                      3h40m
redis-ha                      ClusterIP   None             <none>        6379/TCP,26379/TCP            19m
redis-ha-announce-0           ClusterIP   10.99.205.103    <none>        6379/TCP,26379/TCP            19m
redis-ha-announce-1           ClusterIP   10.106.77.208    <none>        6379/TCP,26379/TCP            19m
redis-ha-announce-2           ClusterIP   10.103.17.152    <none>        6379/TCP,26379/TCP            19m
redis-ha-haproxy              NodePort    10.97.173.27     <none>        6379:31995/TCP                14m

如果修改了配置可以这样更新

helm upgrade redis-ha -f my-values.yaml . --namespace kube-ops

卸载

helm delete redis-ha --purge

5. 验证

[root@master redis-ha]# kubectl exec -it redis-ha-server-0 -n kube-ops sh
Defaulting container name to redis.
Use 'kubectl describe pod/redis-ha-server-0 -n kube-ops' to see all of the containers in this pod.

/data $ redis-cli -h redis-ha-server-0 -p 26379
redis-ha-server-0:26379> PING
PONG

# 查看哨兵信息的redis-master节点
redis-ha-server-0:26379> INFO Sentinel
# Sentinel
sentinel_masters:1
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
sentinel_simulate_failure_flags:0
master0:name=mymaster,status=ok,address=10.99.205.103:6379,slaves=2,sentinels=3  #master节点
redis-ha-server-0:26379> 

10.99.205.103:6379：Redis-Master主节点
slaves=2：Redis-Slaves从节点
sentinels=3：哨兵模式

# 连接redis的master节点
/data $ redis-cli -h 10.99.205.103 -p 6379
10.99.205.103:6379> set name sss
(error) NOAUTH Authentication required.

# 因为前面设置了密码，所以要输入密码
10.99.205.103:6379> auth www19930327
OK
10.99.205.103:6379> INFO Replication
# Replication
role:master         #当前master节点
connected_slaves:2  #两个slaves节点
min_slaves_good_slaves:2
slave0:ip=10.103.17.152,port=6379,state=online,offset=676112,lag=1
slave1:ip=10.106.77.208,port=6379,state=online,offset=676253,lag=0
master_replid:f7f9f4dacc93e18e11e1865c84ab600ea4709b7f
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:676253
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:676253

10.99.205.103:6379> set name sss
OK
10.99.205.103:6379> get name
"sss"

6. 客户端连接

获取刚才输入的key的值

7. 内网连接

如果服务部署和redis部署在一个k8s集群内，因为外网有带宽限制，所以可以用内网ClusterIP+6379连接，没有速度限制

[root@master redis-ha]# kubectl get svc -n course | grep redis
redis-ha              ClusterIP   None             <none>        6379/TCP,26379/TCP   16h
redis-ha-announce-0   ClusterIP   10.99.148.201    <none>        6379/TCP,26379/TCP   16h
redis-ha-announce-1   ClusterIP   10.108.235.127   <none>        6379/TCP,26379/TCP   16h
redis-ha-announce-2   ClusterIP   10.98.90.163     <none>        6379/TCP,26379/TCP   16h
redis-ha-haproxy      NodePort    10.96.129.125    <none>        6379:30992/TCP       16h

redis-ha-announce-0，redis-ha-announce-1和redis-ha-announce-2三个节点一个master节点，两个slave节点，服务中不能连接这三个的ClusterIP，因为一个节点可能随时会变化主从身份，从节点不能写入数据，如果先连接是主节点，后来集群自动换成了从连接，连接的服务就会报错Caused by: io.lettuce.core.RedisCommandExecutionException: READONLY You can't write against a read only replica.，所以应该连接redis-ha-haproxy这个svc，host为10.96.129.125，port为6379