Velero结合minio实现kubernetes业务数据备份与恢复

minio

docker pull minio/minio:RELEASE.2022-08-13T21-54-44Z

docker run -d \
	-p 9000:9000 \
	-p 9090:9090 \
	--name minio \
	--restart=always \
	-v ~/minio/data:/data \
	-e "MINIO_ROOT_USER=Admin" \
	-e "MINIO_ROOT_PASSWORD=Admin@2022" \
	-v /etc/localtime:/etc/localtime \
	-v /etc/timezone:/etc/timezone \
	minio/minio:RELEASE.2022-08-13T21-54-44Z \
	server /data --console-address ":9090"

创建一个名为velerodata的桶

velero

1、安装

wget https://github.com/vmware-tanzu/velero/releases/download/v1.9.1/velero-v1.9.1-linux-amd64.tar.gz

tar xf velero-v1.9.1-linux-amd64.tar.gz

cp velero-v1.9.1-linux-amd64/velero /usr/local/bin

kubectl create ns velero-system

mkdir -p ~/velero

cat > ~/velero/velero-auth.txt << 'EOF'
# 创建访问minio的认证文件
[default]
aws_access_key_id = Admin
aws_secret_access_key = Admin@2022
EOF

velero install --help |grep Image

(default "velero/velero:v1.9.1")

# 设定时区
cat > Dockerfile << 'EOF'
FROM velero/velero:v1.9.1
ENV TZ=Asia/Shanghai
EOF

docker build -t velero/velero:v1.9.1-new .

kubectl set image deployment/velero velero=velero/velero:v1.9.1-new -n velero-system

velero --kubeconfig /root/.kube/config \
  install \
    --provider aws \
    --plugins velero/velero-plugin-for-aws:v1.3.1 \
    --bucket velerodata \
    --secret-file ~/velero/velero-auth.txt \
    --use-volume-snapshots=false \
    --image=velero/velero:v1.9.1-new \
    --namespace velero-system \
    --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://192.168.1.201:9000

2、备份

1、手动备份

DATE=`date +%F-%H-%M-%S`
k8s_ns=kube-flannel

velero backup create ${k8s_ns}-backup-${DATE} \
--include-namespaces ${k8s_ns} \
--kubeconfig=/root/.kube/config \
--namespace velero-system

velero backup get --kubeconfig=/root/.kube/config --namespace velero-system

2、自动备份

velero schedule create --help

k8s_ns=kube-flannel # 需要备份的命名空间

velero schedule create ${k8s_ns}-backup \
--schedule="* * * * *" \
--include-namespaces ${k8s_ns} \
--kubeconfig=/root/.kube/config \
--namespace velero-system

生产：每天0点备份，备份保留24小时

k8s_ns=kube-system

velero schedule create ${k8s_ns}-backup \
--schedule="0 0 * * *" \
--ttl 24h \
--include-namespaces ${k8s_ns} \
--kubeconfig=/root/.kube/config \
--namespace velero-system # 创建备份计划

velero schedule get --kubeconfig=/root/.kube/config --namespace velero-system # 查看备份计划

velero backup get --kubeconfig=/root/.kube/config --namespace velero-system # 查看备份结果

velero schedule delete kube-flannel-backup  --kubeconfig=/root/.kube/config --namespace velero-system # 删除备份计划

3、恢复

velero backup get --kubeconfig=/root/.kube/config --namespace velero-system

velero restore create --from-backup kube-flannel-backup-2022-08-23-13-55-29 --wait --kubeconfig=/root/.kube/config --namespace velero-system

使用velero手动备份整个集群

DATE=`date +%F-%H-%M-%S`

velero backup create k8s-all-ns-backup-${DATE} \
--kubeconfig=/root/.kube/config \
--namespace velero-system

4、卸载velero

安装失败或者从新安装执行
清理删除velero服务（如需删除重新安装时使用）
 
kubectl delete namespace/velero-system clusterrolebinding/velero-velero-system
 
kubectl delete crds -l component=velero

=== 

etcd客户端etcdctl方式备份整个集群

计划任务备份 k8s-etcd

wget https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz

tar xf etcd-v3.5.1-linux-amd64.tar.gz

cp etcd-v3.5.1-linux-amd64/etcdctl /usr/local/bin

ETCDCTL_API=3 etcdctl \
--write-out=table \
--cert="/etc/kubernetes/pki/etcd/server.crt"  \
--key="/etc/kubernetes/pki/etcd/server.key"  \
--cacert="/etc/kubernetes/pki/etcd/ca.crt" \
--endpoints 127.0.0.1:2379 \
endpoint health

mkdir -p ~/crontab
mkdir -p /data/k8s-etcd-backup

cat > ~/crontab/k8s-etcd-pod.sh << 'EOF'
#!/bin/bash
# 每天凌晨0点备份（k8s-etcd-pod）
# 0 0 * * * /bin/sh /root/crontab/k8s-etcd-pod.sh

k8s_etcd_DATE=`date +%F-%H-%M-%S`

ETCDCTL_API=3 /usr/local/bin/etcdctl \
--write-out=table \
--cert="/etc/kubernetes/pki/etcd/server.crt"  \
--key="/etc/kubernetes/pki/etcd/server.key"  \
--cacert="/etc/kubernetes/pki/etcd/ca.crt" \
--endpoints 127.0.0.1:2379 \
snapshot save /data/k8s-etcd-backup/${k8s_etcd_DATE}-snapshot.bak

# 备份保留7天
find /data/k8s-etcd-backup -name "*.bak" -mtime +7 -exec rm -rf {} \;
EOF

[root@master ~]# crontab -l
0 0 * * * sh /root/crontab/k8s-etcd-pod.sh

[root@master ~]# crontab -l
* * * * * sh /root/crontab/k8s-etcd-pod.sh

tail -f /var/spool/mail/root

# 备份保留7天
find /data/k8s-etcd-backup -name "*.bak"

find /data/k8s-etcd-backup -name "*.bak" -mtime +7 -exec rm -rf {} \;

# 备份保留7分钟
find /data/k8s-etcd-backup -name "*.bak"

find /data/k8s-etcd-backup -name "*.bak" -mmin +7 -exec rm -rf {} \;