centos7.9 快速部署单机版对接k8s动态存储-2之redis6.x集群(RDB)

本次使用rbd做StatefulSet 控制器的动态存储

创建rbd池和远程k8s的用户

ceph osd pool create k8s 64
rbd create -p k8s --image rbd-demo2.img --size 2G
rbd info k8s/rbd-demo2.img
ceph auth get-or-create client.kube mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=k8s'
ceph auth get-key client.admin | base64
ceph auth get-key client.kube | base64

创建秘钥对

cat << END>ceph-secret.yaml 
apiVersion: v1
kind: Secret
metadata:
  name: ceph-admin-secret
  namespace: default
data:
  key: QVFEMmdNaGZTQ0VTQ0JBQUc4WS9mTElzVUZoT0RERFl5emZvblE9PQ==
# ceph auth get-key client.admin | base64
type: "kubernetes.io/rbd"

---
apiVersion: v1
kind: Secret
metadata:
  name: ceph-kube-secret
  namespace: default
data:
  key: QVFEdW5jaGZwa2hUT2hBQUxLdk1PSmJkcDNRbm85RjM5RGRobUE9PQ==

# ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
# ceph auth get-key client.kube | base64       
type: "kubernetes.io/rbd"
END

创建sc

cat <<END>rbd-sc.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ceph-rbd
  annotations:
    storageclass.beta.kubernetes.io/is-default-class: "true"
#provisioner: kubernetes.io/rbd
provisioner: ceph.com/rbd
parameters:
  monitors: 172.16.0.71:6789
  adminId: admin
  adminSecretName: ceph-admin-secret
  adminSecretNamespace: default
  pool: k8s
  userId: kube
  userSecretName: ceph-kube-secret
  userSecretNamespace: default
  fsType: ext4
  imageFormat: "2"
  imageFeatures: "layering"
#reclaimPolicy: Retain
END

redis配置

cat <<END>redis.conf
appendonly yes                      
cluster-enabled yes                 
cluster-config-file /var/lib/redis/nodes.conf  
cluster-node-timeout 5000           
dir /var/lib/redis                  
port 6379   
END 
kubectl create configmap redis-conf --from-file=redis.conf

创建deploy

cat <<END>redis-deploy.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis-service 
  labels:
    app: redis
spec:
  ports:
  - port: 6379
    targetPort: 6379
    name: redis-port
  clusterIP: None
  selector:
    app: redis
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
spec:
  selector:
    matchLabels:
      app: redis
  serviceName: "redis-service"
  replicas: 6
  template:
    metadata:
      labels:
        app: redis
    spec:
      terminationGracePeriodSeconds: 20
      containers:
      - name: redis
        image: redis:latest
        imagePullPolicy: Never     
        command:
          - "redis-server"                  #redis启动命令
        args:
          - "/etc/redis/redis.conf"         #redis-server后面跟的参数,换行代表空格
          - "--protected-mode"              #允许外网访问
          - "no"
        # command: redis-server /etc/redis/redis.conf --protected-mode no
        resources:                          #资源
          requests:                         #请求的资源
            cpu: "100m"                     #m代表千分之,相当于0.1 个cpu资源
            memory: "100Mi"                 #内存100m大小
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"              #挂载configmap生成的文件
            mountPath: "/etc/redis"         #挂载到哪个路径下
          - name: "redis-data"              #挂载持久卷的路径
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"                  #引用configMap卷
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"             #创建configMap指定的名称
              path: "redis.conf"            #里面的那个文件--from-file参数后面的文件
  volumeClaimTemplates:
  - metadata:
      name: "redis-data"
      annotations:
        volume.beta.kubernetes.io/storage-class: "ceph-rbd"
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 1Gi
#      volumes:
#        - name: redis-data
#          persistentVolumeClaim:
#            claimName: claim

END

确保IP都可以解析,加awk不打印最后一个字段

kubectl get pods -l app=redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ' | awk 'NF--'

最后创建他们 期间会询问你是否使用上述配置，输入yes即可

kubectl exec -it redis-0 -- redis-cli --cluster create --cluster-replicas 1  $(kubectl get pods -l app=redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ' | awk 'NF--')

登录查看

最暴露一个端口外部客户端连接

cat <<END>node-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis-svc-0
spec:
  selector:
      app: redis
  ports:
    - name: http
      port: 6379
      protocol: TCP
      targetPort: 6379
      nodePort: 26379
    #- name: https
    #  port: 8443
    #  protocol: TCP
    #  targetPort: 443
  type: NodePort

END