一、基于StatefulSet控制器运行Redis Cluster
1、存储环境
在 nfs 存储服务器创建目录
# 安装 nfs 服务
root@haproxy:~# apt install nfs-server
# 创建共享目录
root@k8s-haproxy1:~# mkdir -pv /data/k8sdata/magedu/redis{0,1,2,3,4,5}
mkdir: created directory '/data/k8sdata/magedu/redis0'
mkdir: created directory '/data/k8sdata/magedu/redis1'
mkdir: created directory '/data/k8sdata/magedu/redis2'
mkdir: created directory '/data/k8sdata/magedu/redis3'
mkdir: created directory '/data/k8sdata/magedu/redis4'
mkdir: created directory '/data/k8sdata/magedu/redis5'
# 在末尾添加
root@haproxy:~# vim /etc/exports
/data/k8sdata *(rw,no_root_squash) #读写权限,不做权限映射
# 不重启nfs,使其生效
root@k8s-haproxy1:~# exportfs -arv
exportfs: /etc/exports [1]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exportfs: /etc/exports [2]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/volumes".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exporting *:/data/volumes
exporting *:/data/k8sdata
root@haproxy:~# systemctl enable nfs-server
2、创建 PV
redis-cluster-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv0
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis0
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv1
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv2
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis2
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv3
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis3
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv4
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis4
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv5
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.0.119
path: /data/k8sdata/magedu/redis5
3、部署 Redis 集群
1、准备 redis 配置文件
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# cat redis.conf
appendonly yes #表示开启 AOF(Append Only File)功能,Redis 会将每次写操作的数据写入 AOF 文件,并在重启时从该文件进行数据恢复。
cluster-enabled yes #表示开启 Redis 集群功能。
cluster-config-file /var/lib/redis/nodes.conf #表示 Redis 集群节点配置信息文件的存储路径。
cluster-node-timeout 5000 #表示 Redis 集群中节点间通信的超时时间,单位为毫秒。
dir /var/lib/redis #表示 Redis 数据文件的存储目录。
port 6379 #表示 Redis 监听的端口号。
基于配置文件创建 configmap (或者将配置文件加到镜像里)
# 创建
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl create configmap redis.conf --from-file=redis.conf -n magedu
configmap/redis.conf created
# 验证 configmap
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl get configmaps -n magedu
NAME DATA AGE
kube-root-ca.crt 1 3d8h
redis.conf 1 23s
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl describe configmaps redis.conf -n magedu
Name: redis.conf
Namespace: magedu
Labels: <none>
Annotations: <none>
Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
BinaryData
====
Events: <none>
2、创建集群
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# cat redis.yaml
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: magedu
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis
port: 6379
clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
name: redis-access
namespace: magedu
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis-access
protocol: TCP
port: 6379
targetPort: 6379
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis
namespace: magedu
spec:
serviceName: redis
replicas: 6
selector:
matchLabels:
app: redis
appCluster: redis-cluster
template:
metadata:
labels:
app: redis
appCluster: redis-cluster
spec:
terminationGracePeriodSeconds: 20
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- redis
topologyKey: kubernetes.io/hostname
containers:
- name: redis
image: y73.harbor.com/y73/redis:v4.0.14
command:
- "redis-server"
args:
- "/etc/redis/redis.conf"
- "--protected-mode"
- "no"
resources:
requests:
cpu: "500m"
memory: "500Mi"
ports:
- containerPort: 6379
name: redis
protocol: TCP
- containerPort: 16379
name: cluster
protocol: TCP
volumeMounts:
- name: conf
mountPath: /etc/redis
- name: data
mountPath: /var/lib/redis
volumes:
- name: conf
configMap:
name: redis-conf
items:
- key: redis.conf
path: redis.conf
volumeClaimTemplates:
- metadata:
name: data
namespace: magedu
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 5Gi
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl apply -f redis.yaml
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl get pod -n magedu
3、基于 StatefulSet 控制器的 Pod 正在滚动创建中
4、验证 PVC 状态
4、初始化 Redis 集群
初始化只需要进行一次,redis 4 及之前的版本使用 redis-tribe 工具进行初始化,redis 5 开始使用 redis-cli 。这里临时启动一个 Pod 对整个集群进行初始化
1、创建初始化 Pod
这里临时启动一个 Pod 对整个集群进行初始化
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/redis-cluster# kubectl run -it ubuntu1804 --image=ubuntu:18.04 --restart=Never -n magedu bash
If you don't see a command prompt, try pressing enter.
root@ubuntu1804:/#
root@ubuntu1804:/# apt update
root@ubuntu1804:/# apt install python2.7 python-pip redis-tools dnsutils iputils-ping net-tools
root@ubuntu1804:/# pip install --upgrade pip
root@ubuntu1804:/# pip install redis-trib==0.5.1
# 创建集群
root@ubuntu1804:/# redis-trib.py create \
`dig +short redis-0.redis.magedu.svc.y73.local`:6379 \
`dig +short redis-1.redis.magedu.svc.y73.local`:6379 \
`dig +short redis-2.redis.magedu.svc.y73.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.107.241:6379 checked
INFO:root:Instance at 10.200.36.107:6379 checked
INFO:root:Instance at 10.200.169.146:6379 checked
INFO:root:Add 5462 slots to 10.200.107.241:6379
INFO:root:Add 5461 slots to 10.200.36.107:6379
INFO:root:Add 5461 slots to 10.200.169.146:6379
# 将 redis-3 加入 redis-0
root@ubuntu1804:/# redis-trib.py replicate \
--master-addr `dig +short redis-0.redis.magedu.svc.y73.local`:6379 \
--slave-addr `dig +short redis-3.redis.magedu.svc.y73.local`:6379
# 将 redis-4 加入 redis-1
root@ubuntu1804:/# redis-trib.py replicate \
--master-addr `dig +short redis-1.redis.magedu.svc.y73.local`:6379 \
--slave-addr `dig +short redis-4.redis.magedu.svc.y73.local`:6379
# 将 redis-5 加入 redis-2
root@ubuntu1804:/# redis-trib.py replicate \
--master-addr `dig +short redis-2.redis.magedu.svc.y73.local`:6379 \
--slave-addr `dig +short redis-5.redis.magedu.svc.y73.local`:6379
5、验证 redis 集群状态
查看集群信息
root@k8s-deploy:~# kubectl exec -it -n magedu pods/redis-0 bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@redis-0 /]#
[root@redis-0 /]# redis-cli
127.0.0.1:6379> CLUSTER INFO
127.0.0.1:6379> CLUSTER NODES
在 master 写入数据,然后再 slave 查看数据
(error) MOVED 4768 10.200.107.241:6379
:提示我们去 10.200.107.241:6379 即 redis-1 写数据。我们在 redis-1 写完数据后,去它的 slave 即 redis-4 查看,发现数据就同步过来了
二、基于StatefulSet控制器运行MySQL一主多从
1、镜像准备
# 准备 mysql 镜像
nerdctl pull mysql:5.7.36
nerdctl tag mysql:5.7.36 y73.harbor.com/y73/mysql:5.7.36
nerdctl push y73.harbor.com/y73/mysql:5.7.36
# 准备 xtrabackup 镜像
nerdctl pull yizhiyong/xtrabackup
nerdctl tag yizhiyong/xtrabackup:latest y73.harbor.com/y73/xtrabackup:1.0
nerdctl push y73.harbor.com/y73/xtrabackup:1.0
2、创建 PV
在 nfs 存储服务器创建目录
# 安装 nfs 服务
root@haproxy:~# apt install nfs-server
# 创建共享目录
root@k8s-haproxy1:~# mkdir -p /data/k8sdata/magedu/mysql-datadir-{1,2,3,4,5,6}
# 在末尾添加
root@haproxy:~# vim /etc/exports
/data/k8sdata *(rw,no_root_squash) #读写权限,不做权限映射
# 不重启nfs,使其生效
root@k8s-haproxy1:~# exportfs -arv
exportfs: /etc/exports [1]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
Assuming default behaviour ('no_subtree_c