基于k8s集群的redis-cluster集群

一、提前准备好nfs存储

[root@k8s-master ~]# yum -y install nfs-utils nfs-common rpcbind
[root@k8s-master ~]# mkdir /nfsdata
[root@k8s-master ~]# chmod -R 777 /nfsdata
[root@k8s-master ~]# vim /etc/exports
/nfsdata *(rw,no_root_squash,sync,no_all_squash)
[root@k8s-master ~]# systemctl start rpcbind nfs
[root@k8s-master ~]# systemctl status nfs

# node节点
yum -y install nfs-utils nfs-common

二、制作动态存储

1.安装helm工具

[root@k8s-master ~]#  wget https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
--2024-08-06 15:53:55--  https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
正在解析主机 get.helm.sh (get.helm.sh)... 152.199.39.108, 2606:2800:247:1cb7:261b:1f9c:2074:3c
正在连接 get.helm.sh (get.helm.sh)|152.199.39.108|:443... 已连接。
已发出 HTTP 请求，正在等待回应... 200 OK
长度：14565908 (14M) [application/x-tar]
正在保存至: “helm-v3.10.1-linux-amd64.tar.gz”

100%[=======================================================================>] 14,565,908   157KB/s 用时 46s    

2024-08-06 15:54:43 (312 KB/s) - 已保存 “helm-v3.10.1-linux-amd64.tar.gz” [14565908/14565908])

[root@k8s-master ~]# tar -xzvf helm-v3.10.1-linux-amd64.tar.gz 
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
[root@k8s-master ~]# cp linux-amd64/helm  /usr/bin/
[root@k8s-master ~]# helm version
version.BuildInfo{Version:"v3.10.1", GitCommit:"9f88ccb6aee40b9a0535fcc7efea6055e1ef72c9", GitTreeState:"clean", GoVersion:"go1.18.7"}

2.配置helm下载源

[root@k8s-master ~]# helm repo add stable  http://mirror.azure.cn/kubernetes/charts
"stable" has been added to your repositories
[root@k8s-master ~]# helm repo list
NAME    URL                                     
stable  http://mirror.azure.cn/kubernetes/charts

3.创建storageclass与nfs的目录相关联

[root@k8s-master ~]# helm install nfs-redis stable/nfs-client-provisioner --set nfs.server=192.168.22.134 --set nfs.path=/nfsdata
WARNING: This chart is deprecated
NAME: nfs-redis
LAST DEPLOYED: Tue Aug  6 16:01:14 2024
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
[root@k8s-master ~]# helm list
NAME            NAMESPACE       REVISION        UPDATED                                 STATUS          CHART                            APP VERSION
nfs-redis       default         1               2024-08-06 16:01:14.424540014 +0800 CST deployed        nfs-client-provisioner-1.2.11    3.1.0      
[root@k8s-master ~]# kubectl get pod
NAME                                                READY   STATUS    RESTARTS   AGE
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks   1/1     Running   0          39s

[root@k8s-master ~]# kubectl get sc
NAME         PROVISIONER                                      RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client   cluster.local/nfs-redis-nfs-client-provisioner   Delete          Immediate           true                   53s

三、redis配置文件configmap

[root@k8s-master redis]# vim redis.conf
[root@k8s-master redis]# cat redis.conf 
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
[root@k8s-master redis]#  kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created
[root@k8s-master redis]# kubectl get cf
error: the server doesn't have a resource type "cf"
[root@k8s-master redis]# kubectl get cm
NAME         DATA   AGE
redis-conf   1      19s

四、创建Headless  service

[root@k8s-master redis]# vim headless-service.yaml
[root@k8s-master redis]# cat headless-service.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster
[root@k8s-master redis]# kubectl apply -f headless-service.yaml 
service/redis-service created
[root@k8s-master redis]# kubectl get svc
NAME            TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
kubernetes      ClusterIP   10.96.0.1    <none>        443/TCP    40h
redis-service   ClusterIP   None         <none>        6379/TCP   9s

五、statefulSet运行redis实例

[root@k8s-master redis]# vim statefulset-redis.yaml
[root@k8s-master redis]# cat statefulset-redis.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-app
spec:
  serviceName: "redis-service"
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: daocloud.io/library/redis:6-alpine3.12
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
          - name: redis
            containerPort: 6379
            protocol: "TCP"
          - name: cluster
            containerPort: 16379
            protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
     name: redis-data
    spec:
      accessModes: [ "ReadWriteMany" ]
      storageClassName: "nfs-client"
      resources:
        requests:
          storage: 200M
[root@k8s-master redis]# kubectl apply -f statefulset-redis.yaml 
statefulset.apps/redis-app created
[root@k8s-master redis]# kubectl get pod
NAME                                                READY   STATUS    RESTARTS   AGE
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks   1/1     Running   0          31m
redis-app-0                                         1/1     Running   0          29s
redis-app-1                                         1/1     Running   0          26s
redis-app-2                                         1/1     Running   0          21s
redis-app-3                                         1/1     Running   0          17s
redis-app-4                                         1/1     Running   0          12s
redis-app-5                                         1/1     Running   0          8s
[root@k8s-master redis]# kubectl get statefulset
NAME        READY   AGE
redis-app   6/6     68s

六、验证唯一访问标识可用性

        每个Pod都会得到集群内的⼀个DNS域名，格式为

(podname).(servicename).(namespace).svc.cluster.local

例如： redis-app-0.redis-service.default.svc.cluster.local

           redis-app-1.redis-service.default.svc.cluster.local

# 创建一个pod用于测试，测试完即可删除
[root@k8s-master redis]# vim busybox.yaml
[root@k8s-master redis]# cat busybox.yaml 
---
apiVersion: v1
kind: Pod
metadata:
  name: busybox
spec:
  containers:
  - name: busybox
    image: daocloud.io/library/busybox
    stdin: true
    tty: true
[root@k8s-master redis]# kubectl apply -f busybox.yaml 
pod/busybox created
[root@k8s-master redis]# kubectl get pod -o wide
NAME                                                READY   STATUS    RESTARTS   AGE     IP           NODE        NOMINATED NODE   READINESS GATES
busybox                                             1/1     Running   0          2m56s   10.244.1.9   k8s-node1   <none>           <none>
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks   1/1     Running   0          51m     10.244.2.4   k8s-node2   <none>           <none>
redis-app-0                                         1/1     Running   0          20m     10.244.1.4   k8s-node1   <none>           <none>
redis-app-1                                         1/1     Running   0          20m     10.244.2.5   k8s-node2   <none>           <none>
redis-app-2                                         1/1     Running   0          20m     10.244.1.5   k8s-node1   <none>           <none>
redis-app-3                                         1/1     Running   0          19m     10.244.2.6   k8s-node2   <none>           <none>
redis-app-4                                         1/1     Running   0          19m     10.244.1.6   k8s-node1   <none>           <none>
redis-app-5                                         1/1     Running   0          19m     10.244.2.7   k8s-node2   <none>           <none>

[root@k8s-master redis]# kubectl exec -it busybox /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ping redis-app-0.redis-service.default.svc.cluster.local
PING redis-app-0.redis-service.default.svc.cluster.local (10.244.1.4): 56 data bytes
64 bytes from 10.244.1.4: seq=0 ttl=64 time=0.132 ms
64 bytes from 10.244.1.4: seq=1 ttl=64 time=0.062 ms
64 bytes from 10.244.1.4: seq=2 ttl=64 time=0.074 ms
^C
--- redis-app-0.redis-service.default.svc.cluster.local ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.062/0.089/0.132 ms
/ # ping redis-app-1.redis-service.default.svc.cluster.local
PING redis-app-1.redis-service.default.svc.cluster.local (10.244.2.5): 56 data bytes
64 bytes from 10.244.2.5: seq=0 ttl=62 time=2.076 ms
64 bytes from 10.244.2.5: seq=1 ttl=62 time=0.796 ms
^C
--- redis-app-1.redis-service.default.svc.cluster.local ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.796/1.456/2.108 ms
/ # ping redis-app-2.redis-service.default.svc.cluster.local
PING redis-app-2.redis-service.default.svc.cluster.local (10.244.1.5): 56 data bytes
64 bytes from 10.244.1.5: seq=0 ttl=64 time=0.167 ms
64 bytes from 10.244.1.5: seq=1 ttl=64 time=0.067 ms
64 bytes from 10.244.1.5: seq=2 ttl=64 time=0.061 ms
^C
--- redis-app-2.redis-service.default.svc.cluster.local ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.061/0.098/0.167 ms
/ # 

 查看PV和PVC

[root@k8s-master redis]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                            STORAGECLASS   REASON   AGE
pvc-04c6e1bc-709f-4a77-bd36-7cd8c23d4ca1   200M       RWX            Delete           Bound    default/redis-data-redis-app-1   nfs-client              21m
pvc-1650521c-ba23-4502-b8e8-c260359a2d27   200M       RWX            Delete           Bound    default/redis-data-redis-app-0   nfs-client              21m
pvc-63d54f9b-7be4-4ce7-a72b-caa7bedde507   200M       RWX            Delete           Bound    default/redis-data-redis-app-2   nfs-client              21m
pvc-652e836c-f426-4629-8600-d542f4269a52   200M       RWX            Delete           Bound    default/redis-data-redis-app-3   nfs-client              21m
pvc-b1eee630-b2fc-4423-be6c-32278d6a8018   200M       RWX            Delete           Bound    default/redis-data-redis-app-5   nfs-client              21m
pvc-efc54629-d01d-45ac-9c42-37ecbe34189a   200M       RWX            Delete           Bound    default/redis-data-redis-app-4   nfs-client              21m
[root@k8s-master redis]# kubectl get pvc
NAME                     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
redis-data-redis-app-0   Bound    pvc-1650521c-ba23-4502-b8e8-c260359a2d27   200M       RWX            nfs-client     21m
redis-data-redis-app-1   Bound    pvc-04c6e1bc-709f-4a77-bd36-7cd8c23d4ca1   200M       RWX            nfs-client     21m
redis-data-redis-app-2   Bound    pvc-63d54f9b-7be4-4ce7-a72b-caa7bedde507   200M       RWX            nfs-client     21m
redis-data-redis-app-3   Bound    pvc-652e836c-f426-4629-8600-d542f4269a52   200M       RWX            nfs-client     21m
redis-data-redis-app-4   Bound    pvc-efc54629-d01d-45ac-9c42-37ecbe34189a   200M       RWX            nfs-client     21m
redis-data-redis-app-5   Bound    pvc-b1eee630-b2fc-4423-be6c-32278d6a8018   200M       RWX            nfs-client     21m

七、redis集群初始化

        由于Redis集群必须在所有节点启动后才能进⾏初始化，这⾥，我们专⻔启动⼀个Ubuntu的容器，可以在该容器中安装Redis-tribe，进⽽初始化Redis集群；

1）创建Ubuntu容器并配置源

[root@k8s-master redis]# kubectl run -it ubuntu --image=daocloud.io/library/ubuntu:artful-20170619 --restart=Never /bin/bash
If you don't see a command prompt, try pressing enter.
root@ubuntu:/#
# 添加阿里云的Ubuntu源，并安装基本的软件环境
root@ubuntu:/# root@ubuntu:/# cat > /etc/apt/sources.list << EOF
> deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
> EOF
root@ubuntu:/# apt-get update
  ......
root@ubuntu:/# echo $?
0
root@ubuntu:/# apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils

2）安装redis-trib

root@ubuntu:/# pip install redis-trib==0.5.1

3）创建只有master节点的集群

root@ubuntu:/# redis-trib.py create \
 `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
 `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
 `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379

4）为每个master添加slave

root@ubuntu:/# redis-trib.py replicate \
 --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379
root@ubuntu:/# redis-trib.py replicate \
 --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379
root@ubuntu:/# redis-trib.py replicate \
 --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
 --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379

5） 连接到任意一个redis pod中检验一下

[root@k8s-master redis]# kubectl exec -it redis-app-5 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> CLUSTER NODES
42beeed3837a809034874d7415c307e91e2daac8 10.244.2.5:6379@16379 master - 0 1722937557103 1 connected 0-5461
c06e4e94182f72196bcf524e948c6265a8fcfb87 10.244.1.5:6379@16379 master - 0 1722937556000 4 connected 10923-16383
9bcbf04efa0705a10ab9c80b5379f346f2e32c38 10.244.2.7:6379@16379 myself,slave c06e4e94182f72196bcf524e948c6265a8fcfb87 0 1722937555000 4 connected
79ebf7ed87288fe7b23065c5e8911878b8dde0e4 10.244.2.6:6379@16379 slave 3413f32da2cb4e177b2a6c57ffd1171850c039c2 0 1722937556296 2 connected
f62ab0f4d9e3ac82163d63310716cc460ac31898 10.244.1.6:6379@16379 slave 42beeed3837a809034874d7415c307e91e2daac8 0 1722937555792 1 connected
3413f32da2cb4e177b2a6c57ffd1171850c039c2 10.244.1.4:6379@16379 master - 0 1722937556802 2 connected 5462-10922
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:4
cluster_my_epoch:4
cluster_stats_messages_ping_sent:654
cluster_stats_messages_pong_sent:582
cluster_stats_messages_sent:1236
cluster_stats_messages_ping_received:581
cluster_stats_messages_pong_received:654
cluster_stats_messages_meet_received:1
cluster_stats_messages_received:1236
127.0.0.1:6379> 

八、创建用于访问Service

        还需要创建⼀个Service，专⽤于为Redis集群提供访问和负载均衡；

[root@k8s-master redis]# vim redis-access-service.yaml
[root@k8s-master redis]# cat redis-access-service.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  selector:
    app: redis
    appCluster: redis-cluster
[root@k8s-master redis]# kubectl apply -f redis-access-service.yaml 
service/redis-access-service created
[root@k8s-master redis]# kubectl get svc
NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
kubernetes             ClusterIP   10.96.0.1       <none>        443/TCP    42h
redis-access-service   ClusterIP   10.103.21.164   <none>        6379/TCP   11s
redis-service          ClusterIP   None            <none>        6379/TCP   103m
[root@k8s-master redis]# kubectl get ep
NAME                                             ENDPOINTS                                                     AGE
cluster.local-nfs-redis-nfs-client-provisioner   <none>                                                        116m
kubernetes                                       192.168.22.134:6443                                           42h
redis-access-service                             10.244.1.4:6379,10.244.1.5:6379,10.244.1.6:6379 + 3 more...   25s
redis-service                                    10.244.1.4:6379,10.244.1.5:6379,10.244.1.6:6379 + 3 more...   103m