二进制k8s集群etcd备份和恢复
当前环境
##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.203 node03
192.168.48.54 nfs
## keepalived的vip
192.168.48.66
IP Hostname CPU Memory 192.168.48.101 master01 2 4G 192.168.48.102 master02 2 4G 192.168.48.103 master03 2 4G 192.168.48.201 node01 2 4G 192.168.48.202 node02 2 4G 192.168.48.203 node03 2 4G 192.168.48.54 nfs 2 4G
软件 版本 kubernetes 1.15.3 docker-ce 19.03 calico 3.8 etcd 3.3.13 CNI 0.8.1 coredns 1.4.0
当前版本
[root@master01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
master01 Ready master 29d v1.15.3
master02 Ready master 29d v1.15.3
master03 Ready master 29d v1.15.3
node01 Ready node 29d v1.15.3
node02 Ready node 29d v1.15.3
node03 Ready node 77m v1.15.3
etcd操作
–keys-only 默认为true,只显示key,如果设置为false,会显示key的所有值. –prefix 默认为true可以看到所有的子目录.
查看版本
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt version
etcdctl version: 3.3.13
API version: 3.3
查看etcd集群
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt member list -w table
+------------------+---------+----------+-----------------------------+-----------------------------+
| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS |
+------------------+---------+----------+-----------------------------+-----------------------------+
| cf07d604d88be6a | started | master01 | https://192.168.48.101:2380 | https://192.168.48.101:2379 |
| 6c8995c4a94f5a29 | started | master03 | https://192.168.48.103:2380 | https://192.168.48.103:2379 |
| 8cf70d11a9c8d0c5 | started | master02 | https://192.168.48.102:2380 | https://192.168.48.102:2379 |
+------------------+---------+----------+-----------------------------+-----------------------------+
查看pod资源
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt get /registry/pods --prefix --keys-only
/registry/pods/default/counter
/registry/pods/kube-system/calico-kube-controllers-7bd78b474d-r2tdd
/registry/pods/kube-system/calico-node-cfckb
/registry/pods/kube-system/calico-node-hhtlb
/registry/pods/kube-system/calico-node-kzqg6
/registry/pods/kube-system/calico-node-msmv2
/registry/pods/kube-system/calico-node-p2kvc
/registry/pods/kube-system/calico-node-q8tr4
/registry/pods/kube-system/coredns-5c6c9cf6c8-ppmf4
/registry/pods/kube-system/coredns-5c6c9cf6c8-x2sj2
查看service资源
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt get /registry/services --prefix --keys-only
/registry/services/endpoints/default/kubernetes
/registry/services/endpoints/default/nfs-k8s
/registry/services/endpoints/kube-system/kube-controller-manager
/registry/services/endpoints/kube-system/kube-dns
/registry/services/endpoints/kube-system/kube-scheduler
/registry/services/specs/default/kubernetes
/registry/services/specs/kube-system/kube-dns
删除操作
[root@master01 ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
counter 1/1 Running 1 10d
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt del /registry/pods/default/counter
1
[root@master01 ~]# kubectl get pod
No resources found.
备份etcd
[root@master01 etcd]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt snapshot save /etc/etcd/20190911backup.db
Snapshot saved at /etc/etcd/20190911backup.db
[root@master01 etcd]# cd /etc/etcd/
[root@master01 etcd]# ll
total 3812
-rw-r--r-- 1 root root 3891232 Sep 11 23:09 20190911backup.db
-rw-r--r-- 1 root root 1506 Aug 12 22:02 config.yml
-rw-r--r-- 1 root root 1516 Aug 12 22:07 etcd.config.yml
恢复etcd
关闭kube-apiserver
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
ssh ${MasterArray[$name]} "systemctl stop kube-apiserver.service"
done
关闭etcd集群
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
ssh ${MasterArray[$name]} "systemctl stop etcd.service"
done
删除etcd集群的数据
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
ssh ${MasterArray[$name]} "rm -rf /var/lib/etcd/"
done
将etcd备份数据发到etcd节点
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
scp /etc/etcd/20190911backup.db ${MasterArray[$name]}:/etc/etcd/20190911backup.db
done
恢复etcd
[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore /etc/etcd/20190911backup.db
[root@master02 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore /etc/etcd/20190911backup.db
[root@master03 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore /etc/etcd/20190911backup.db
启动etcd
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
ssh ${MasterArray[$name]} "systemctl start etcd.service"
done
启动kube-apiserver
for name in ${!MasterArray[@]};do
echo "--- $name ${MasterArray[$name]} ---"
ssh ${MasterArray[$name]} "systemctl start kube-apiserver.service"
done