第七周记录

一、基于StatefulSet控制器运行Redis Cluster

1、存储环境

在 nfs 存储服务器创建目录

# 安装 nfs 服务
root@haproxy:~# apt install nfs-server

# 创建共享目录
root@k8s-haproxy1:~# mkdir -pv /data/k8sdata/magedu/redis{0,1,2,3,4,5}
mkdir: created directory '/data/k8sdata/magedu/redis0'
mkdir: created directory '/data/k8sdata/magedu/redis1'
mkdir: created directory '/data/k8sdata/magedu/redis2'
mkdir: created directory '/data/k8sdata/magedu/redis3'
mkdir: created directory '/data/k8sdata/magedu/redis4'
mkdir: created directory '/data/k8sdata/magedu/redis5'


# 在末尾添加
root@haproxy:~# vim /etc/exports
/data/k8sdata *(rw,no_root_squash)   #读写权限，不做权限映射

# 不重启nfs，使其生效
root@k8s-haproxy1:~# exportfs -arv
exportfs: /etc/exports [1]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
  Assuming default behaviour ('no_subtree_check').
  NOTE: this default has changed since nfs-utils version 1.0.x

exportfs: /etc/exports [2]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/volumes".
  Assuming default behaviour ('no_subtree_check').
  NOTE: this default has changed since nfs-utils version 1.0.x

exporting *:/data/volumes
exporting *:/data/k8sdata


root@haproxy:~# systemctl enable nfs-server

2、创建 PV

redis-cluster-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv0
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis0 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv1
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv2
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis2 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv3
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis3 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv4
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis4 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-cluster-pv5
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.0.119
    path: /data/k8sdata/magedu/redis5 

3、部署 Redis 集群

1、准备 redis 配置文件

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# cat redis.conf
appendonly yes  #表示开启 AOF（Append Only File）功能，Redis 会将每次写操作的数据写入 AOF 文件，并在重启时从该文件进行数据恢复。
cluster-enabled yes  #表示开启 Redis 集群功能。
cluster-config-file /var/lib/redis/nodes.conf  #表示 Redis 集群节点配置信息文件的存储路径。
cluster-node-timeout 5000  #表示 Redis 集群中节点间通信的超时时间，单位为毫秒。
dir /var/lib/redis  #表示 Redis 数据文件的存储目录。
port 6379  #表示 Redis 监听的端口号。

基于配置文件创建 configmap （或者将配置文件加到镜像里）

# 创建
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl create configmap redis.conf --from-file=redis.conf -n magedu 
configmap/redis.conf created

# 验证 configmap
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl get configmaps -n magedu 
NAME               DATA   AGE
kube-root-ca.crt   1      3d8h
redis.conf         1      23s

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl describe configmaps redis.conf -n magedu 
Name:         redis.conf
Namespace:    magedu
Labels:       <none>
Annotations:  <none>

Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379


BinaryData
====

Events:  <none>

2、创建集群

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# cat redis.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: magedu
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis
    port: 6379
  clusterIP: None
  
---
apiVersion: v1
kind: Service
metadata:
  name: redis-access
  namespace: magedu
  labels:
    app: redis
spec:
  selector:
    app: redis
    appCluster: redis-cluster
  ports:
  - name: redis-access
    protocol: TCP
    port: 6379
    targetPort: 6379

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: magedu
spec:
  serviceName: redis
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: y73.harbor.com/y73/redis:v4.0.14
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "500m"
            memory: "500Mi"
        ports:
        - containerPort: 6379
          name: redis
          protocol: TCP
        - containerPort: 16379
          name: cluster
          protocol: TCP
        volumeMounts:
        - name: conf
          mountPath: /etc/redis
        - name: data
          mountPath: /var/lib/redis
      volumes:
      - name: conf
        configMap:
          name: redis-conf
          items:
          - key: redis.conf
            path: redis.conf
  volumeClaimTemplates:
  - metadata:
      name: data
      namespace: magedu
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 5Gi

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl apply -f redis.yaml
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl get pod -n magedu 

3、基于 StatefulSet 控制器的 Pod 正在滚动创建中

4、验证 PVC 状态

4、初始化 Redis 集群

初始化只需要进行一次，redis 4 及之前的版本使用 redis-tribe 工具进行初始化，redis 5 开始使用 redis-cli 。这里临时启动一个 Pod 对整个集群进行初始化

1、创建初始化 Pod

这里临时启动一个 Pod 对整个集群进行初始化

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl run -it ubuntu1804 --image=ubuntu:18.04 --restart=Never -n magedu bash
If you don't see a command prompt, try pressing enter.
root@ubuntu1804:/# 
root@ubuntu1804:/# apt update
root@ubuntu1804:/# apt install python2.7 python-pip redis-tools dnsutils iputils-ping net-tools
root@ubuntu1804:/# pip install --upgrade pip
root@ubuntu1804:/# pip install redis-trib==0.5.1

# 创建集群
root@ubuntu1804:/# redis-trib.py create \
 `dig +short redis-0.redis.magedu.svc.y73.local`:6379 \
 `dig +short redis-1.redis.magedu.svc.y73.local`:6379 \
 `dig +short redis-2.redis.magedu.svc.y73.local`:6379  
 
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.107.241:6379 checked
INFO:root:Instance at 10.200.36.107:6379 checked
INFO:root:Instance at 10.200.169.146:6379 checked
INFO:root:Add 5462 slots to 10.200.107.241:6379
INFO:root:Add 5461 slots to 10.200.36.107:6379
INFO:root:Add 5461 slots to 10.200.169.146:6379


# 将 redis-3 加入 redis-0
root@ubuntu1804:/# redis-trib.py replicate \
 --master-addr `dig +short redis-0.redis.magedu.svc.y73.local`:6379 \
 --slave-addr `dig +short redis-3.redis.magedu.svc.y73.local`:6379  

# 将 redis-4 加入 redis-1
root@ubuntu1804:/# redis-trib.py replicate \
 --master-addr `dig +short redis-1.redis.magedu.svc.y73.local`:6379 \
 --slave-addr `dig +short redis-4.redis.magedu.svc.y73.local`:6379  

# 将 redis-5 加入 redis-2
root@ubuntu1804:/# redis-trib.py replicate \
 --master-addr `dig +short redis-2.redis.magedu.svc.y73.local`:6379 \
 --slave-addr `dig +short redis-5.redis.magedu.svc.y73.local`:6379  

5、验证 redis 集群状态

查看集群信息

root@k8s-deploy:~# kubectl exec -it -n magedu pods/redis-0 bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@redis-0 /]# 
[root@redis-0 /]# redis-cli 
127.0.0.1:6379> CLUSTER INFO

127.0.0.1:6379> CLUSTER NODES

在 master 写入数据，然后再 slave 查看数据

(error) MOVED 4768 10.200.107.241:6379 ：提示我们去 10.200.107.241:6379 即 redis-1 写数据。我们在 redis-1 写完数据后，去它的 slave 即 redis-4 查看，发现数据就同步过来了

二、基于StatefulSet控制器运行MySQL一主多从

1、镜像准备

# 准备 mysql 镜像
nerdctl pull mysql:5.7.36
nerdctl tag mysql:5.7.36 y73.harbor.com/y73/mysql:5.7.36
nerdctl push y73.harbor.com/y73/mysql:5.7.36

# 准备 xtrabackup 镜像
nerdctl pull yizhiyong/xtrabackup
nerdctl tag yizhiyong/xtrabackup:latest y73.harbor.com/y73/xtrabackup:1.0
nerdctl push y73.harbor.com/y73/xtrabackup:1.0

2、创建 PV

在 nfs 存储服务器创建目录

# 安装 nfs 服务
root@haproxy:~# apt install nfs-server

# 创建共享目录
root@k8s-haproxy1:~# mkdir -p /data/k8sdata/magedu/mysql-datadir-{1,2,3,4,5,6}

# 在末尾添加
root@haproxy:~# vim /etc/exports
/data/k8sdata *(rw,no_root_squash)   #读写权限，不做权限映射

# 不重启nfs，使其生效
root@k8s-haproxy1:~# exportfs -arv
exportfs: /etc/exports [1]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
  Assuming default behaviour ('no_subtree_