k8s部署redis一主两从三哨兵

完整的脚本下载，前往gitee：https://gitee.com/qiaodaimadewangcai/study-notes/tree/master/k8s/k8s部署redis主从+哨兵/yaml

一、部署思路

1. 一共6个容器，1个主节点、2个从节点、3个哨兵节点。
2. 编写namespace脚本，创建专门的namespace
3. 编写configmap脚本，分别将主节点、从节点、哨兵的配置写在里面
4. 编写secret脚本，将redis的密码保存在里面
5. 编写storageClass脚本，用于自动创建磁盘资源
6. 编写service脚本，1个提供redis之间的网络，1个提供哨兵之间的网络，1个对外提供服务
7. 编写statefulSet脚本，1个提供redis主从服务，1个提供哨兵集群服务

二、部署

部署说明

软件名称	软件版本
redis	v6.2.5
kubernetes	v20.10.17
docker	v1.23.10

部署条件

1. 有个k8s集群
2. k8s集群集成了nfs之类的作为存储抽象

1、编写namespace脚本

01-redis-namespace.yaml

apiVersion: v1
#创建Namespace类型资源
kind: Namespace
metadata:
  #资源名称
  name: redis
  labels:
    app: redis

相关命令

#执行命令
kubectl apply -f 01-redis-namespace.yaml
#查看命名空间
kubectl get ns

2、编写configmap脚本

02-redis-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
data:
  #这里定义了多个数据信息
  master.conf: |
    # Master配置
    requirepass redisPassword
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    replica-read-only no
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  slave.conf: |
    # Slave配置
    replicaof redis-0.redis-svc 6379
    masterauth redisPassword
    requirepass redisPassword
    replica-read-only yes
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  sentinel.conf: |
    # 哨兵配置
    port 26379
    daemonize no
    pidfile "/var/run/redis-sentinel.pid"
    logfile ""
    dir "/data"
    sentinel monitor master0 redis-0.redis-svc 6379 2
    sentinel auth-pass master0 a123456!
    sentinel down-after-milliseconds master0 30000
    sentinel parallel-syncs master0 1
    sentinel failover-timeout master0 180000
    acllog-max-len 128
    sentinel deny-scripts-reconfig yes
    sentinel resolve-hostnames yes
    sentinel announce-hostnames no
    protected-mode no
    user default on nopass sanitize-payload ~* &* +@all

相关命令

#执行命令
kubectl apply -f 02-redis-configmap.yaml
#查看redis命名空间下的configmap
kubectl get cm -n redis
#查看redis命名空间下名为redis的configmap详情
kubectl describe configmap redis -n redis

3、编写secret脚本

03-redis-secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  namespace: redis
  labels:
    app: redis
#Opaque 类型的数据是一个 map 类型，要求value是base64编码。
type: Opaque
data:
  redisPassword: YTEyMzQ1NiE= #123456转成base64 echo -n "a123456!" | base64

相关命令

#执行命令
kubectl apply -f 03-redis-secret.yaml
#查看redis命名空间下的configmap
kubectl get secret -n redis
#查看redis命名空间下名为redis-secret的secret详情
kubectl describe secret redis-secret -n redis

4、编写StorageClass脚本

采用StorageClass+NFS方式作为网络存储，后续会自动生成pvc和pv

所有的k8s节点上都要安装nfs，NFS搭建略过，信息如下

IP: 192.168.56.80
Export PATH: /mnt

1）编写ServiceAccount、ClusterRole、ClusterRoleBinding、Role、RoleBinding脚本管理NFS

04-redis-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  namespace: redis
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: redis
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: redis
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: redis
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

相关命令

#执行命令
kubectl apply -f 04-redis-rbac.yaml

2）编写StorageClass脚本

05-redis-nfs-storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: redis-nfs-storage #这里的名称要和provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
volumeBindingMode: WaitForFirstConsumer
parameters:
  archiveOnDelete: "true" #pvc被删除了也需要进行存档

相关命令

#执行命令
kubectl apply -f 05-redis-nfs-storageclass.yaml
#查看StorageClass信息
kubectl get sc

3）编写nfs-provisioner的Deployment脚本

所有的k8s节点上都要安装nfs，不然这段就会出问题无法运行

06-redis-nfs-provisioner-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  namespace: redis  #与RBAC文件中的namespace保持一致
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: redis-nfs-storage  #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
            - name: NFS_SERVER
              value: 192.168.56.80   #NFS Server IP地址
            - name: NFS_PATH  
              value: /mnt    #NFS挂载卷
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.56.80  #NFS Server IP地址
            path: /mnt     #NFS 挂载卷

相关命令

#执行命令
kubectl apply -f 06-redis-nfs-provisioner-deployment.yaml

5、编写Service脚本

07-redis-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: redis-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
  	#匹配带有app: redis标签的pod
    app: redis
  clusterIP: None
  ports:
  - name: redis
    port: 6379
-----------------------------------------
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
  	#匹配带有app: redis标签的pod
    app: redis-sentinel
  clusterIP: None
  ports:
  - name: redis
    port: 26379
------------------------------------------
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc-nodeport
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    app: redis-sentinel
  type: NodePort
  ports:
  #集群内访问  <集群内部ip>:26379  会被代理到对应pod的26379端口
  #集群外访问  <k8s的ip>:30379   会被代理到对应pod的26379端口
  - name: redisSentinel
    #集群内部之间访问使用的端口
    port: 26379
    #pod上需要被代理的端口
    targetPort: 26379
    #集群外访问使用的端口，范围固定 30000 ~ 32767
    nodePort: 30379

相关命令

#执行命令
kubectl apply -f 07-redis-service.yaml
#查看redis命名空间下service信息
kubectl get svc -n redis

6、编写redis的StatefulSet脚本

08-redis-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    matchLabels:
      app: redis
  #与redis-service.yaml中的保持一致
  serviceName: redis-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis
    spec:
      initContainers:
      - name: init-redis
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          #从pod的hostname中通过正则获取序号，如果没有截取到就退出程序
          ordinal=`hostname | awk -F"-" '{print $2}'`
          if [ ${ordinal} -eq 0 ]; then
            # 如果Pod的序号为0，说明它是Master节点
            cp /mnt/config-map/master.conf /conf/redis.conf
          else
            # 否则，拷贝ConfigMap里的Slave的配置文件
            cp /mnt/config-map/slave.conf /conf/redis.conf
          fi
          sed -i "s/redisPassword/${REDIS_PASSWORD}/g"  /conf/redis.conf
        env:
        - name: REDIS_PASSWORD
          valueFrom:
            secretKeyRef:
              name: redis-secret
              key: redisPassword
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis
        image: redis:6.2.5
        command: ["redis-server"]
        args:
          - "/conf/redis.conf"
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 2Gi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 5Gi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi

相关命令

#执行命令
kubectl apply -f 08-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis
#查看redis主从信息（redis客户端中使用）
info replication

7、编写哨兵的StatefulSet脚本

09-redis-sentinel-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-sentinel
  namespace: redis
  labels:
    app: redis-sentinel
spec:
  selector:
    matchLabels:
      app: redis-sentinel
  #与redis-service.yaml中的保持一致
  serviceName: redis-sentinel-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis-sentinel
    spec:
      initContainers:
      - name: init-redis-sentinel
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          cp /mnt/config-map/sentinel.conf /conf/redis-sentinel.conf
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis-sentinel
        image: redis:6.2.5
        command: ["redis-sentinel"]
        args:
          - "/conf/redis-sentinel.conf"
        ports:
        - name: redis-sentinel
          containerPort: 26379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 2Gi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 10Mi

相关命令

#执行命令
kubectl apply -f 09-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis
#查看redis哨兵信息（redis客户端中使用）
sentinel sentinels master0