mysql k8s 有状态集群_Kubernetes集群部署有状态主从容器mysql5.7集群

0 前言

本文主要的工作是解决容器集群的有状态化的问题以及进行验证相关技术。

1 相关环境

名称

版本

Kubernetes

v1.18.6

RKE

1.1.4

网络插件

calico

2.拓扑图

3.创建测试ns

apiVersion: v1

kind: Namespace

metadata:

name: mysql

labels:

app: mysql

4.创建Storgeclass

apiVersion: v1

kind: PersistentVolume

metadata:

name: example-mysql-pv

spec:

capacity:

storage: 3Gi

volumeMode: Filesystem

accessModes:

- ReadWriteOnce

persistentVolumeReclaimPolicy: Delete

storageClassName: local-storage

local:

path: /data/mysql/

nodeAffinity:

required:

nodeSelectorTerms:

- matchExpressions:

- key: kubernetes.io/hostname

operator: In

values:

- node(name) #此处务必是节点名称，也就是kubectl get node获得的节点名称

5.创建ConfigMap

apiVersion: v1

kind: ConfigMap

metadata:

name: mysql

namespace: mysql

labels:

app: mysql

data:

master.cnf: |

# Master节点配置

[mysqld]

log-bin=mysqllog

skip-name-resolve

slave.cnf: |

# Slave节点配置

[mysqld]

super-read-only

skip-name-resolve

log-bin=mysql-bin

replicate-ignore-db=mysql

apiVersion: v1

kind: Secret

metadata:

name: mysql-secret

namespace: mysql

labels:

app: mysql

type: Opaque

data:

password: aGVsbG93b3JsZA== # helloworld

6.部署无头服务

apiVersion: v1

kind: Service

metadata:

name: mysql

namespace: mysql

labels:

app: mysql

spec:

ports:

- name: mysql

port: 3306

clusterIP: None

selector:

app: mysql

7. 部署有状态容器集群

apiVersion: apps/v1

kind: StatefulSet

metadata:

name: mysql

namespace: mysql

labels:

app: mysql

spec:

selector:

matchLabels:

app: mysql

serviceName: mysql

replicas: 2

template:

metadata:

labels:

app: mysql

spec:

initContainers:

- name: init-mysql

image: mysql:5.7

env:

- name: MYSQL_ROOT_PASSWORD

valueFrom:

secretKeyRef:

name: mysql-secret

key: password

command:

- bash

- "-c"

- |

set -ex

# 从 Pod 的序号，生成 server-id

[[ $(hostname) =~ -([0-9]+)$ ]] || exit 1

ordinal=${BASH_REMATCH[1]}

echo [mysqld] > /mnt/conf.d/server-id.cnf

# 由于 server-id 不能为 0，因此给 ID 加 100 来避开它

echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf

# 如果 Pod 的序号为 0，说明它是 Master 节点，从 ConfigMap 里把 Master 的配置文件拷贝到 /mnt/conf.d 目录下

# 否则，拷贝 ConfigMap 里的 Slave 的配置文件

if [[ ${ordinal} -eq 0 ]]; then

cp /mnt/config-map/master.cnf /mnt/conf.d

else

cp /mnt/config-map/slave.cnf /mnt/conf.d

fi

volumeMounts:

- name: conf

mountPath: /mnt/conf.d

- name: config-map

mountPath: /mnt/config-map

- name: clone-mysql

image: gcr.io/google-samples/xtrabackup:1.0

env:

- name: MYSQL_ROOT_PASSWORD

valueFrom:

secretKeyRef:

name: mysql-secret

key: password

command:

- bash

- "-c"

- |

set -ex

# 拷贝操作只需要在第一次启动时进行，所以数据已经存在则跳过

[[ -d /var/lib/mysql/mysql ]] && exit 0

# Master 节点(序号为 0)不需要这个操作

[[ $(hostname) =~ -([0-9]+)$ ]] || exit 1

ordinal=${BASH_REMATCH[1]}

[[ $ordinal == 0 ]] && exit 0

# 使用 ncat 指令，远程地从前一个节点拷贝数据到本地

ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql

# 执行 --prepare，这样拷贝来的数据就可以用作恢复了

xtrabackup --prepare --target-dir=/var/lib/mysql

volumeMounts:

- name: data

mountPath: /var/lib/mysql

subPath: mysql

- name: conf

mountPath: /etc/mysql/conf.d

containers:

- name: mysql

image: mysql:5.7

env:

# - name: MYSQL_ALLOW_EMPTY_PASSWORD

# value: "1"

- name: MYSQL_ROOT_PASSWORD

valueFrom:

secretKeyRef:

name: mysql-secret

key: password

ports:

- name: mysql

containerPort: 3306

volumeMounts:

- name: data

mountPath: /var/lib/mysql

subPath: mysql

- name: conf

mountPath: /etc/mysql/conf.d

resources:

requests:

cpu: 500m

memory: 1Gi

livenessProbe:

exec:

command: ["mysqladmin", "ping", "-uroot", "-p${MYSQL_ROOT_PASSWORD}"]

initialDelaySeconds: 30

periodSeconds: 10

timeoutSeconds: 5

readinessProbe:

exec:

command: ["mysqladmin", "ping", "-uroot", "-p${MYSQL_ROOT_PASSWORD}"]

initialDelaySeconds: 5

periodSeconds: 2

timeoutSeconds: 1

- name: xtrabackup

image: gcr.io/google-samples/xtrabackup:1.0

ports:

- name: xtrabackup

containerPort: 3307

env:

- name: MYSQL_ROOT_PASSWORD

valueFrom:

secretKeyRef:

name: mysql-secret

key: password

command:

- bash

- "-c"

- |

set -ex

cd /var/lib/mysql

# 从备份信息文件里读取 MASTER_LOG_FILE 和 MASTER_LOG_POS 这 2 个字段的值，用来拼装集群初始化 SQL

if [[ -f xtrabackup_slave_info ]]; then

# 如果 xtrabackup_slave_info 文件存在，说明这个备份数据来自于另一个 Slave 节点

# 这种情况下，XtraBackup 工具在备份的时候，就已经在这个文件里自动生成了 "CHANGE MASTER TO" SQL 语句

# 所以，只需要把这个文件重命名为 change_master_to.sql.in，后面直接使用即可

mv xtrabackup_slave_info change_master_to.sql.in

# 所以，也就用不着 xtrabackup_binlog_info 了

rm -f xtrabackup_binlog_info

elif [[ -f xtrabackup_binlog_info ]]; then

# 如果只是存在 xtrabackup_binlog_info 文件，说明备份来自于 Master 节点，就需要解析这个备份信息文件，读取所需的两个字段的值

[[ $(cat xtrabackup_binlog_info) =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1

rm xtrabackup_binlog_info

# 把两个字段的值拼装成 SQL，写入 change_master_to.sql.in 文件

echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\

MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in

fi

# 如果存在 change_master_to.sql.in，就意味着需要做集群初始化工作

if [[ -f change_master_to.sql.in ]]; then

# 但一定要先等 MySQL 容器启动之后才能进行下一步连接 MySQL 的操作

echo "Waiting for mysqld to be ready(accepting connections)"

until mysql -h 127.0.0.1 -uroot -p${MYSQL_ROOT_PASSWORD} -e "SELECT 1"; do sleep 1; done

echo "Initializing replication from clone position"

# 将文件 change_master_to.sql.in 改个名字

# 防止这个 Container 重启的时候，因为又找到了 change_master_to.sql.in，从而重复执行一遍初始化流程

mv change_master_to.sql.in change_master_to.sql.orig

# 使用 change_master_to.sql.orig 的内容，也就是前面拼装的 SQL，组成一个完整的初始化和启动 Slave 的 SQL 语句

mysql -h 127.0.0.1 -uroot -p${MYSQL_ROOT_PASSWORD} << EOF

$(< change_master_to.sql.orig),

MASTER_HOST='mysql-0.mysql.mysql',

MASTER_USER='root',

MASTER_PASSWORD='${MYSQL_ROOT_PASSWORD}',

MASTER_CONNECT_RETRY=10;

START SLAVE;

EOF

fi

# 使用 ncat 监听 3307 端口。

# 它的作用是，在收到传输请求的时候，直接执行 xtrabackup --backup 命令，备份 MySQL 的数据并发送给请求者

exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \

"xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root --password=${MYSQL_ROOT_PASSWORD}"

volumeMounts:

- name: data

mountPath: /var/lib/mysql

subPath: mysql

- name: conf

mountPath: /etc/mysql/conf.d

volumes:

- name: conf

emptyDir: {}

- name: config-map

configMap:

name: mysql

volumeClaimTemplates:

- metadata:

name: data

spec:

accessModes:

- "ReadWriteOnce"

storageClassName: local-storage

resources:

requests:

storage: 3Gi

8 制作mycat-1.5.1镜像

#**注意：本地打的镜像，需要从本地复制jdk以及mycat文件，不要直接复制**

FROM centos:8

RUN mkdir -p /root/soft

RUN mkdir -p /usr/local/java

RUN mkdir -p /usr/local/mycat

COPY ./jdk1.8.0_261 /usr/local/java/

ENV JAVA_HOME /usr/local/java

ENV JRE_HOME ${JAVA_HOME}/jre

ENV CLASSPATH .:${JAVA_HOME}/lib:${JRE_HOME}/lib

ENV PATH ${JAVA_HOME}/bin:$PATH

COPY ./mycat /usr/local/mycat

RUN cd /usr/local/mycat/

RUN mv /usr/local/mycat/conf/schema.xml /usr/local/mycat/conf/schema.xml.backup

RUN mv /usr/local/mycat/conf/server.xml /usr/local/mycat/conf/server.xml.backup

COPY ./schema.xml /usr/local/mycat/conf/schema.xml

COPY ./server.xml /usr/local/mycat/conf/server.xml

9 运行mycat镜像

kubectl run mycat --image=mycat:1.5.1