kubernetes (statefulSet+PV+PVC)部署redis集群(数据持久化)

kubernetes 部署redis集群(数据持久化)

• 其中ceph可以改为nfs创建pv/pvc

1-创建pv用于挂载数据

1.1-创建文件夹

此处采用ceph挂载文件,直接存储在指定文件夹即可

mkdir pv{0..5}

1.2-创建pv

cd /home/yanyonglong/deployments/redis-cluster
cat >> pv.yaml << EOF
kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv0
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv0"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - iotedge03
          - iotedge02
          - iotedge01
---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv1
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv1"
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - iotedge03
                - iotedge02
                - iotedge01
---

kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv2
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv2"
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - iotedge03
                - iotedge02
                - iotedge01

---

kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv3
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv3"
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - iotedge03
                - iotedge02
                - iotedge01

---

kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv4
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv4"
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - iotedge03
                - iotedge02
                - iotedge01

---

kind: PersistentVolume
apiVersion: v1
metadata:
  name: redis-pv5
  labels:
    type: local
spec:
  storageClassName: redis-pv
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  local:
    path: "/data/edgebox/iotedge/remote/redis/pv5"
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - iotedge03
                - iotedge02
                - iotedge01


EOF

# 创建
kubectl apply -f pv.yaml
kubectl get pv -o wide

1.3-创建pvc

cat >> pvc.yaml << EOF
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-0
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-1
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-2
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-3
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-4
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv

---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: redis-data-redis-app-5
  namespace: "default"
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: redis-pv


EOF

# 创建
kubectl apply -f pvc.yaml
kubectl get pvc -o wide

2-创建配置

# 创建配置
cat >> conf.yaml << EOF
kind: ConfigMap
apiVersion: v1
metadata:
  name: redis-config
data:
  redis.conf: |
    appendonly yes
    cluster-enabled yes
    cluster-config-file /var/lib/redis/nodes.conf
    cluster-node-timeout 5000
    dir /var/lib/redis
    port 6379

EOF 

3-创建redis-headle

cat >> headless-server.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster

EOF
# 创建
kubectl apply -f headless-server.yaml
kubectl get svc

4-创建redis副本

cat >> redis-stateful.yaml << EOF
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-app
spec:
  serviceName: "redis-service"
  replicas: 6
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: redis
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-data
      labels:
        app: redis
    spec:
      accessModes: [ "ReadWriteMany" ]
      storageClassName: redis-pv
      resources:
        requests:
          storage: 10G

EOF

# 创建
kubectl apply -f redis-stateful.yaml

5-启动集群

# 通过新建一个pod,用这个pod来启动集群(启动后可以删除)
kubectl run -it ubuntu --image=ubuntu --restart=Never /bin/bash

# 安装
apt-get update
apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils
pip install redis-trib==0.5.1


# 创建master
redis-trib.py create \
  `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379



# 为master分配slave
redis-trib.py replicate \
  --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379


# 配置完成后进去redis查看状态
kubectl exec -it redis-app-2 /bin/bash
redis-cli -c
cluster ndoes

6-创建redis的access方便访问

# 暴露nodePort
cat >> redis-access-service.yaml << EOF
apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  type: NodePort
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
    nodePort: 30079
  selector:
    app: redis
    appCluster: redis-cluster

EOF

# 创建代理service
kubectl apply -f redis-access-service.yaml

7-测试使用Python操作redis

pip3 install redis-py-cluster

# 在只需要访问代理节点即可
from rediscluster import RedisCluster
addr = [{"host":"10.56.1.11","port":30079}]
rc = RedisCluster(startup_nodes=addr)
rc.set("aaa", "111")
data = rc.get("aaa")

操作图片示例