Kubernetes基于helm部署Redis哨兵集群

注：当前Kubernetes版本是1.30.2，helm版本3.15.2，redis版本是7.2.5

一、添加并更新Helm仓库

helm repo add bitnami  https://charts.bitnami.com/bitnami
helm repo update bitnami

二、下载并解压redis的Chart

helm pull bitnami/redis
tar -xf redis-19.6.4.tgz

三、修改values.yml
下面为修改后的示例：
我已经删除了多余的注释和默认的配置，仅保留修改后的必要的内容

global:
  redis:
    # passwd: ""
    password: "123456"
  compatibility:
    openshift:
      adaptSecurityContext: auto

clusterDomain: cluster.local
useHostnames: true
nameResolutionThreshold: 5
nameResolutionTimeout: 5
diagnosticMode:
  enabled: false
  command:
    - sleep
  args:
    - infinity

image:
  registry: registry.cn-hangzhou.aliyuncs.com
  repository: zhaoll/redis
  tag: 7.2.5-debian-12-r4
  pullPolicy: IfNotPresent

architecture: replication
auth:
  enabled: true
  sentinel: true
  # passwd: ""
  password: "123456"
  usePasswordFileFromSecret: true
commonConfiguration: |-
  appendonly no
  save ""
master:
  count: 1
  revisionHistoryLimit: 10
  disableCommands:
    - FLUSHDB
    - FLUSHALL
  enableServiceLinks: true
  containerPorts:
    redis: 6379
  startupProbe:
    enabled: false
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  livenessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 5
  readinessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 1
    successThreshold: 1
    failureThreshold: 5
  podSecurityContext:
    enabled: true
    fsGroupChangePolicy: Always
    sysctls: []
    supplementalGroups: []
    fsGroup: 1001
  containerSecurityContext:
    enabled: true
    seLinuxOptions: {}
    runAsUser: 1001
    runAsGroup: 1001
    runAsNonRoot: true
    allowPrivilegeEscalation: false
    readOnlyRootFilesystem: true
    seccompProfile:
      type: RuntimeDefault
    capabilities:
      drop: ["ALL"]
  kind: StatefulSet
  updateStrategy:
    type: RollingUpdate
  minReadySeconds: 0
  automountServiceAccountToken: false
  shareProcessNamespace: false
  podAffinityPreset: ""
  podAntiAffinityPreset: soft
  persistence:
    enabled: true
    path: /data
    storageClass: "nfs-storage"
    accessModes:
      - ReadWriteOnce
    size: 1Gi
  persistentVolumeClaimRetentionPolicy:
    enabled: false
    whenScaled: Retain
    whenDeleted: Retain
  service:
    type: ClusterIP
    portNames:
      redis: "tcp-redis"
    ports:
      redis: 6379
    nodePorts:
      redis: ""
    externalTrafficPolicy: Cluster
    internalTrafficPolicy: Cluster
  terminationGracePeriodSeconds: 30
  serviceAccount:
    create: true
    name: ""
    automountServiceAccountToken: false
    annotations: {}
  pdb:
    create: true
    minAvailable: ""
    maxUnavailable: ""
replica:
  automountServiceAccountToken: true

sentinel:
  enabled: true
  image:
    registry: registry.cn-hangzhou.aliyuncs.com
    repository: zhaoll/redis-sentinel
    tag: 7.2.5-debian-12-r4
    pullPolicy: IfNotPresent
  masterSet: mymaster
  quorum: 2
  getMasterTimeout: 90
  automateClusterRecovery: false
  redisShutdownWaitFailover: true
  downAfterMilliseconds: 60000
  failoverTimeout: 180000
  parallelSyncs: 1
  enableServiceLinks: true
  containerPorts:
    sentinel: 26379
  startupProbe:
    enabled: true
    initialDelaySeconds: 10
    periodSeconds: 10
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 22
  livenessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 10
    timeoutSeconds: 5
    successThreshold: 1
    failureThreshold: 6
  readinessProbe:
    enabled: true
    initialDelaySeconds: 20
    periodSeconds: 5
    timeoutSeconds: 1
    successThreshold: 1
    failureThreshold: 6
  persistence:
    enabled: true
    storageClass: "nfs-storage"
    accessModes:
      - ReadWriteOnce
    size: 100Mi
  persistentVolumeClaimRetentionPolicy:
    enabled: false
    whenScaled: Retain
    whenDeleted: Retain
  containerSecurityContext:
    enabled: true
    seLinuxOptions: {}
    runAsUser: 1001
    runAsGroup: 1001
    runAsNonRoot: true
    allowPrivilegeEscalation: false
    readOnlyRootFilesystem: true
    seccompProfile:
      type: RuntimeDefault
    capabilities:
      drop: ["ALL"]
  service:
    # type: ClusterIP
    type: NodePort
    ports:
      redis: 6379
      sentinel: 26379
    nodePorts:
      redis: ""
      sentinel: ""
    externalTrafficPolicy: Cluster
    #clusterIP: ""
    # createMaster: false
    createMaster: true
    headless:
      annotations: {}

  masterService:
    # enabled: false
    enabled: true
    type: ClusterIP
    ports:
      redis: 6379
    nodePorts:
      redis: ""
    externalTrafficPolicy: ""
    extraPorts: []
    clusterIP: ""
  terminationGracePeriodSeconds: 30

serviceBindings:
  enabled: false
networkPolicy:
  enabled: true
  allowExternal: true
  allowExternalEgress: true
  extraIngress: []
  extraEgress: []
  ingressNSMatchLabels: {}
  ingressNSPodMatchLabels: {}
  metrics:
    allowExternal: true
    ingressNSMatchLabels: {}
    ingressNSPodMatchLabels: {}
podSecurityPolicy:
  create: false
  enabled: false
rbac:
  # create: false
  create: true
  rules: []
serviceAccount:
  create: true
  name: ""
  automountServiceAccountToken: false
  annotations: {}
pdb: {}

kubectl:
  image:
    registry: registry.cn-hangzhou.aliyuncs.com
    repository: zhaoll/kubectl
    tag: 1.30.3-debian-12-r4
    digest: ""
    pullPolicy: IfNotPresent
    pullSecrets: []
  command: ["/opt/bitnami/scripts/kubectl-scripts/update-master-label.sh"]
  containerSecurityContext:
    enabled: true
    seLinuxOptions: {}
    runAsUser: 1001
    runAsGroup: 1001
    runAsNonRoot: true
    allowPrivilegeEscalation: false
    readOnlyRootFilesystem: true
    seccompProfile:
      type: RuntimeDefault
    capabilities:
      drop: ["ALL"]

三、执行安装命令

[root@master1 redis]# helm upgrade redis-sentinel -f demo.yaml bitnami/redis
Release "redis-sentinel" has been upgraded. Happy Helming!
NAME: redis-sentinel
LAST DEPLOYED: Thu Aug  8 00:24:52 2024
NAMESPACE: default
STATUS: deployed
REVISION: 2
TEST SUITE: None
NOTES:
CHART NAME: redis
CHART VERSION: 19.6.4
APP VERSION: 7.2.5

** Please be patient while the chart is being deployed **

Redis® can be accessed via port 6379 on the following DNS name from within your cluster:

    redis-sentinel.default.svc.cluster.local for read only operations

For read/write operations, first access the Redis® Sentinel cluster, which is available in port 26379 using the same domain name above.



To get your password run:

    export REDIS_PASSWORD=$(kubectl get secret --namespace default redis-sentinel -o jsonpath="{.data.redis-password}" | base64 -d)

To connect to your Redis® server:

1. Run a Redis® pod that you can use as a client:

   kubectl run --namespace default redis-client --restart='Never'  --env REDIS_PASSWORD=$REDIS_PASSWORD  --image registry.cn-hangzhou.aliyuncs.com/zhaoll/redis:7.2.5-debian-12-r4 --command -- sleep infinity

   Use the following command to attach to the pod:

   kubectl exec --tty -i redis-client \
   --namespace default -- bash

2. Connect using the Redis® CLI:
   REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h redis-sentinel -p 6379 # Read only operations
   REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h redis-sentinel -p 26379 # Sentinel access

To connect to your database from outside the cluster execute the following commands:

    export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
    export NODE_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services redis-sentinel)
    REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h $NODE_IP -p $NODE_PORT

WARNING: There are "resources" sections in the chart not set. Using "resourcesPreset" is not recommended for production. For production installations, please set the following values according to your workload needs:
  - replica.resources
  - sentinel.resources
+info https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/

⚠ SECURITY WARNING: Original containers have been substituted. This Helm chart was designed, tested, and validated on multiple platforms using a specific set of Bitnami and Tanzu Application Catalog containers. Substituting other containers is likely to cause degraded security and performance, broken chart features, and missing environment variables.

Substituted images detected:
  - registry.cn-hangzhou.aliyuncs.com/zhaoll/redis:7.2.5-debian-12-r4
  - registry.cn-hangzhou.aliyuncs.com/zhaoll/redis-sentinel:7.2.5-debian-12-r4
  - registry.cn-hangzhou.aliyuncs.com/zhaoll/kubectl:1.30.3-debian-12-r4

五、按照上面的提示创建测试Pod

kubectl run --namespace default redis-client --restart='Never'  --env REDIS_PASSWORD=$REDIS_PASSWORD  --image registry.cn-hangzhou.aliyuncs.com/zhaoll/redis:7.2.5-debian-12-r4 --command -- sleep infinity

六，查看资源状态

[root@master1 redis]# kubectl get pod
NAME                                      READY   STATUS    RESTARTS      AGE
busybox                                   1/1     Running   3 (54m ago)   18d
nfs-client-provisioner-66f4984679-ngct9   1/1     Running   1 (54m ago)   11d
redis-client                              1/1     Running   0             11s
redis-sentinel-node-0                     3/3     Running   0             10m
redis-sentinel-node-1                     3/3     Running   0             9m50s
redis-sentinel-node-2                     3/3     Running   0             9m6s
[root@master1 redis]# kubectl get sts
NAME                  READY   AGE
redis-sentinel-node   3/3     10m
[root@master1 redis]# kubectl get svc
NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                                          AGE
kubernetes                ClusterIP   10.96.0.1        <none>        443/TCP                                                          18d
redis-sentinel            NodePort    10.96.181.148    <none>        30001:30001/TCP,30000:30000/TCP,26379:30101/TCP,6379:31875/TCP   10m
redis-sentinel-headless   ClusterIP   None             <none>        6379/TCP,26379/TCP                                               11m
redis-sentinel-master     ClusterIP   10.96.159.88     <none>        6379/TCP                                                         10m
redis-sentinel-node-0     NodePort    10.104.205.175   <none>        30002:30002/TCP,30003:30003/TCP,26379:31815/TCP,6379:31261/TCP   10m
redis-sentinel-node-1     NodePort    10.107.238.46    <none>        30004:30004/TCP,30005:30005/TCP,26379:32724/TCP,6379:31371/TCP   10m
redis-sentinel-node-2     NodePort    10.106.125.38    <none>        30006:30006/TCP,30007:30007/TCP,26379:32079/TCP,6379:30926/TCP   10m
[root@master1 redis]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                         STORAGECLASS   VOLUMEATTRIBUTESCLASS   REASON   AGE
pvc-1b90691d-69ec-4d71-aca5-656bf1f04478   8Gi        RWO            Retain           Bound    default/redis-data-redis-sentinel-node-2      nfs-storage    <unset>                          25m
pvc-41e22544-02f9-4d4a-bf00-7e5096ff1fc8   8Gi        RWO            Retain           Bound    default/redis-data-redis-sentinel-node-0      nfs-storage    <unset>                          37m
pvc-7e0030d3-9db9-4c04-b519-cb6ac488c33f   100Mi      RWO            Retain           Bound    default/sentinel-data-redis-sentinel-node-2   nfs-storage    <unset>                          25m
pvc-966cfe6e-b670-4f10-b0ea-ca2c6887c6ba   100Mi      RWO            Retain           Bound    default/sentinel-data-redis-sentinel-node-0   nfs-storage    <unset>                          37m
pvc-9deae579-882b-4201-bb2c-59a7e694ca9c   100Mi      RWO            Retain           Bound    default/sentinel-data-redis-sentinel-node-1   nfs-storage    <unset>                          25m
pvc-a80af212-4b6a-4d07-b208-50a4e8252471   1Gi        RWX            Retain           Bound    default/nfs-storage                           nfs-storage    <unset>                          11d
pvc-e22af47b-79a1-4556-bb3c-42cf812ecaff   8Gi        RWO            Retain           Bound    default/redis-data-redis-sentinel-node-1      nfs-storage    <unset>                          25m
[root@master1 redis]# 

七、连接Redis

[root@master1 redis]# kubectl exec --tty -i redis-client \
>    --namespace default -- bash
I have no name!@redis-client:/$ 
I have no name!@redis-client:/$ REDISCLI_AUTH=123456 redis-cli -h redis-sentinel -p 26379
redis-sentinel:26379> info
# Server
redis_version:7.2.5
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:86a33a678b1e6059
redis_mode:sentinel
os:Linux 5.4.278-1.el7.elrepo.x86_64 x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:12.2.0
process_id:1
process_supervised:no
run_id:e80e6bf5c43b41b99d98e1bb5b6aab75fc7b82cb
tcp_port:26379
server_time_usec:1723048628359678
uptime_in_seconds:580
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:11772596
executable:/redis-server
config_file:/opt/bitnami/redis-sentinel/etc/sentinel.conf
io_threads_active:0
listener0:name=tcp,bind=*,bind=-::*,port=26379

# Clients
connected_clients:3
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0

# Stats
total_connections_received:172
total_commands_processed:1954
instantaneous_ops_per_sec:2
total_net_input_bytes:117117
total_net_output_bytes:41966
total_net_repl_input_bytes:0
total_net_repl_output_bytes:0
instantaneous_input_kbps:0.13
instantaneous_output_kbps:0.01
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:7
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:2088
total_writes_processed:1918
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:4
reply_buffer_expands:0
eventloop_cycles:18810
eventloop_duration_sum:5913329
eventloop_duration_cmd_sum:758332
instantaneous_eventloop_cycles_per_sec:30
instantaneous_eventloop_duration_usec:188
acl_access_denied_auth:0
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

# CPU
used_cpu_sys:3.047426
used_cpu_user:0.076262
used_cpu_sys_children:0.019636
used_cpu_user_children:0.022721
used_cpu_sys_main_thread:3.046267
used_cpu_user_main_thread:0.076156

# Sentinel
sentinel_masters:1
sentinel_tilt:0
sentinel_tilt_since_seconds:-1
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
sentinel_simulate_failure_flags:0
master0:name=mymaster,status=ok,address=redis-sentinel-node-0.default:30003,slaves=2,sentinels=3

我们连接的是哨兵节点，使用info命令从最后一行可以看到，当前有1个master，2个slave和3个sentinel。
查看master节点信息：

[root@master1 redis]# kubectl exec --tty -i redis-client    --namespace default -- bash
I have no name!@redis-client:/$ REDISCLI_AUTH=123456 redis-cli -h redis-sentinel -p 26379
redis-sentinel:26379> sentinel master mymaster
 1) "name"
 2) "mymaster"
 3) "ip"
 4) "redis-sentinel-node-0.default"
 5) "port"
 6) "30003"
 7) "runid"
 8) "6fca9cef640f2e19adb27cd0c7c6244a090a444a"
 9) "flags"
10) "master"
11) "link-pending-commands"
12) "0"
13) "link-refcount"
14) "1"
15) "last-ping-sent"
16) "0"
17) "last-ok-ping-reply"
18) "511"
19) "last-ping-reply"
20) "511"
21) "down-after-milliseconds"
22) "60000"
23) "info-refresh"
24) "923"
25) "role-reported"
26) "master"
27) "role-reported-time"
28) "1497452"
29) "config-epoch"
30) "0"
31) "num-slaves"
32) "2"
33) "num-other-sentinels"
34) "2"
35) "quorum"
36) "2"
37) "failover-timeout"
38) "180000"
39) "parallel-syncs"
40) "1"
41) "client-reconfig-script"
42) "/opt/bitnami/scripts/start-scripts/push-master-label.sh"

可以看到当前master节点是redis-sentinel-node-0.default。
查看slave节点：

redis-sentinel:26379> sentinel slaves mymaster
1)  1) "name"
    2) "redis-sentinel-node-2.default:30007"
    3) "ip"
    4) "redis-sentinel-node-2.default"
    5) "port"
    6) "30007"
    7) "runid"
    8) "a20f6f644bc7c3502c83e4b8d15c7573b38aff2e"
    9) "flags"
   10) "slave"
   11) "link-pending-commands"
   12) "0"
   13) "link-refcount"
   14) "1"
   15) "last-ping-sent"
   16) "0"
   17) "last-ok-ping-reply"
   18) "38"
   19) "last-ping-reply"
   20) "38"
   21) "down-after-milliseconds"
   22) "60000"
   23) "info-refresh"
   24) "8540"
   25) "role-reported"
   26) "slave"
   27) "role-reported-time"
   28) "1541090"
   29) "master-link-down-time"
   30) "0"
   31) "master-link-status"
   32) "ok"
   33) "master-host"
   34) "redis-sentinel-node-0.default"
   35) "master-port"
   36) "30003"
   37) "slave-priority"
   38) "100"
   39) "slave-repl-offset"
   40) "397389"
   41) "replica-announced"
   42) "1"
2)  1) "name"
    2) "redis-sentinel-node-1.default:30005"
    3) "ip"
    4) "redis-sentinel-node-1.default"
    5) "port"
    6) "30005"
    7) "runid"
    8) "c27e47f38cf195715a5f6648bec39458e932c8c1"
    9) "flags"
   10) "slave"
   11) "link-pending-commands"
   12) "-2"
   13) "link-refcount"
   14) "1"
   15) "last-ping-sent"
   16) "0"
   17) "last-ok-ping-reply"
   18) "38"
   19) "last-ping-reply"
   20) "38"
   21) "down-after-milliseconds"
   22) "60000"
   23) "info-refresh"
   24) "9086"
   25) "role-reported"
   26) "slave"
   27) "role-reported-time"
   28) "1571883"
   29) "master-link-down-time"
   30) "0"
   31) "master-link-status"
   32) "ok"
   33) "master-host"
   34) "redis-sentinel-node-0.default"
   35) "master-port"
   36) "30003"
   37) "slave-priority"
   38) "100"
   39) "slave-repl-offset"
   40) "397389"
   41) "replica-announced"
   42) "1"