k8s部署redis三主三从（nfs）

@王先生1

已于 2024-05-21 10:10:18 修改

阅读量146

点赞数 1

文章标签： k8s redis nfs 三主三从

于 2023-11-17 17:10:57 首次发布

本文链接：https://blog.csdn.net/qq_44637753/article/details/134441449

版权

k8s部署三主三从（nfs）

1、搭建nfs-server
2、创建存储类：
3、创建pv卷（1..6）
4、编辑configmap简单配置
5、创建redis-svc
6、创建redis-sts
7、apply
- 如果创建失败需要删除pv、pvc。
8、创建redis-cluster集群：
9、验证集群是否创建成功。

1、搭建nfs-server

ubuntu：

apt install nfs-kernel-server -y 
所有服务器安装nfs客户端
apt install nfs-common

centos:

yum install nfs-server -y

mkdir /data

cat /etc/exports

/data *(async,insecure,no_root_squash,no_subtree_check,rw)

2、创建存储类：

cat > sc.yaml << EOF
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: redis-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsimer
EOF

kubectl apply -f sc.yaml

kubectl get sc 


mkdir -p /data/redis/pv{1..6}

3、创建pv卷（1…6）

apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv1
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv1
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv2
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv2
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv3
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv3
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv4
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv4
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv5
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv5
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-nfs-pv6
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: redis-storage
  nfs:
    server: 192.168.220.129
    path: /data/redis/pv6
#  nodeAffinity:
#    required:
#      nodeSelectorTerms:
#      - matchExpressions:
#        - key: kubernetes.io/hostname
#          operator: In
#          values:
#          - k8s-master01

4、编辑configmap简单配置

cat > cm.yaml << EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-conf
  namespace: local
data:
  redis.conf:  |
    daemonize no
    appendonly yes
    cluster-enabled yes
    cluster-config-file /etc/redis/redis.conf
    cluster-node-timeout 5000
    dir /data/
    port 6379
    masterauth 123456
    requirepass 123456
EOF

redis-conf需要注意以下配置
protected-mode no
daemonize no

kubectl create configmap redis-conf --from-file=redis.conf=redis.conf -n local

bind 0.0.0.0
protected-mode no    
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /data/redis.pid
loglevel notice
logfile /data/redis-server.log
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /data/
masterauth abc123456
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
requirepass abc123456
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file redis.conf
cluster-node-timeout 15000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes

5、创建redis-svc

apiVersion: v1
kind: Service
metadata:
  name: redis-headless
  namespace: local
  labels:
    app: redis
spec:
  clusterIP: None
  ports:
  - name: redis
    port: 6379
  selector:
    app: redis
---
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  namespace: local
  labels:
    app: redis
spec:
  type: NodePort
  ports:
  - name: redis-client
    port: 6379
    nodePort: 16379
  selector:
    app: redis

6、创建redis-sts

cat sts.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: local
spec:
  serviceName: redis-headless
  replicas: 6
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      - name: redis
        image: redis:7.0.12
        imagePullPolicy: IfNotPersent
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"      ##redis.conf已定义
          - "no"
#          - "--cluster-annouce-ip"     ## #这个参数是为了解决pod重启ip变了之后,redis集群状态无法自动同步问题（未实验）
#          - "$(POD_IP)"
#        env:
#        - name: POD_IP
#          valueFrom:
#            fieldRdf:
#              fieldPath: status.podIP
        resources:
          requests:
            cpu: "500m"
            memory: "500Mi"
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis/redis.conf"    
          - name: "redis-data"
            mountPath: "/data/"
      volumes:
        - name: "redis-conf"
          configMap:
            name: "redis-conf"
            items:
              - key: "redis.conf"
                path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 1Gi
      storageClassName: "redis-storage"

7、apply

kubectl apply -f sts.yaml
在这里插入图片描述

如果创建失败需要删除pv、pvc。

kubectl patch pv redis-nfs-pv1 -p '{"metadata":{"finalizers":null}}'
kubectl patch pv redis-nfs-pv2 -p '{"metadata":{"finalizers":null}}'
kubectl patch pv redis-nfs-pv3 -p '{"metadata":{"finalizers":null}}'
kubectl patch pv redis-nfs-pv4 -p '{"metadata":{"finalizers":null}}'
kubectl patch pv redis-nfs-pv5 -p '{"metadata":{"finalizers":null}}'
kubectl patch pv redis-nfs-pv6 -p '{"metadata":{"finalizers":null}}'

kubectl patch pvc redis-data-redis-0 -n local -p '{"metadata":{"finalizers":null}}'
kubectl patch pvc redis-data-redis-1 -n local -p '{"metadata":{"finalizers":null}}'
kubectl patch pvc redis-data-redis-2 -n local -p '{"metadata":{"finalizers":null}}'
kubectl patch pvc redis-data-redis-3 -n local -p '{"metadata":{"finalizers":null}}'
kubectl patch pvc redis-data-redis-4 -n local -p '{"metadata":{"finalizers":null}}'
kubectl patch pvc redis-data-redis-5 -n local -p '{"metadata":{"finalizers":null}}'


rm -rf pv{1..6}/*

8、创建redis-cluster集群：

所有查询在local命名空间下，端口6379密码abc123456

kubectl exec -it redis-0 -n local -- redis-cli -a abc123456 --cluster create --cluster-replicas 1  $(kubectl get pods -l app=redis -n local -o jsonpath='{range.items[*]}{.status.podIP}:6379 {end}')

在这里插入图片描述

9、验证集群是否创建成功。

kubectl exec -it redis-0 -n local -- redis-cli -a abc123456 --cluster check  $(kubectl get pods -l app=redis -n local -o jsonpath='{range.items[0]}{.status.podIP}:6379 {end}')

在这里插入图片描述
pod内部创建集群

kubectl get pod -l app=redis -n local -owide

kubectl exec -it redis-0 -n local -- redis-cli -a abc123456 --cluster create 172.20.0.12:6379 172.20.1.187:6379  172.20.2.248:6379


kubectl exec -it redis-0 -n local -- redis-cli -a abc123456 --cluster create   --cluster-replicas 1 172.20.0.12:6379 172.20.1.187:6379  172.20.2.248:6379  172.20.3.85:6379  172.20.5.214:6379  172.20.4.186:6379

kubectl exec -it redis-0 -n local -- bash

重启pod验证集群是否正常。

kubectl exec -it redis-0 -n local -- bash