kuberntes 系统使用 etcd 存储所有数据,本文档介绍部署一个三节点高可用 etcd 集群的步骤,这三个节点复用 kubernetes 集群机器k8s-master、k8s-node-1、k8s-node-2.
1.准备etcd软件包并分发etcd文件:
[root@k8s-master ~]# cd /usr/local/src [root@k8s-master src]#wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz [root@k8s-master src]# tar zxf etcd-v3.2.18-linux-amd64.tar.gz [root@k8s-master src]# cd etcd-v3.2.18-linux-amd64 [root@k8s-master etcd-v3.2.18-linux-amd64]# cp etcd etcdctl /opt/kubernetes/bin/ [root@k8s-master etcd-v3.2.18-linux-amd64]# scp etcd etcdctl 10.200.3.106:/opt/kubernetes/bin/ [root@k8s-master etcd-v3.2.18-linux-amd64]# scp etcd etcdctl 10.200.3.107:/opt/kubernetes/bin/
2.创建 etcd 证书签名请求:
[root@k8s-master ~]# cd /usr/local/src/ssl
[root@k8s-master ssl]# cat > etcd-csr.json <<EOF { "CN": "etcd", "hosts": [ "127.0.0.1", "10.200.3.105", "10.200.3.106", "10.200.3.107" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "k8s", "OU": "System" } ] } EOF
3.生成 etcd 证书和私钥:
[root@k8s-master ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \ -ca-key=/opt/kubernetes/ssl/ca-key.pem \ -config=/opt/kubernetes/ssl/ca-config.json \ -profile=kubernetes etcd-csr.json | cfssljson -bare etcd 会生成以下证书文件 [root@k8s-master ssl]# ls -l etcd* -rw-r--r-- 1 root root 1045 Mar 5 11:27 etcd.csr -rw-r--r-- 1 root root 257 Mar 5 11:25 etcd-csr.json -rw------- 1 root root 1679 Mar 5 11:27 etcd-key.pem -rw-r--r-- 1 root root 1419 Mar 5 11:27 etcd.pem
4.将证书移动到/opt/kubernetes/ssl目录下:
[root@k8s-master ssl]# cp etcd*.pem /opt/kubernetes/ssl [root@k8s-master ssl]# scp etcd*.pem 10.200.3.106:/opt/kubernetes/ssl [root@k8s-master ssl]# scp etcd*.pem 10.200.3.107:/opt/kubernetes/ssl [root@k8s-master ssl]# rm -f etcd.csr etcd-csr.json
5.设置ETCD 配置文件
root@k8s-master ssl]#cat > /opt/kubernetes/cfg/etcd.conf <<EOF #[member] ETCD_NAME="k8s-master" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_SNAPSHOT_COUNTER="10000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" ETCD_LISTEN_PEER_URLS="https://10.200.3.105:2380" ETCD_LISTEN_CLIENT_URLS="https://10.200.3.105:2379,https://127.0.0.1:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" #ETCD_CORS="" #[cluster] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.105:2380" # if you use different ETCD_NAME (e.g. test), # set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380" ETCD_INITIAL_CLUSTER_STATE="new" ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster" ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.105:2379" #[security] CLIENT_CERT_AUTH="true" ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" PEER_CLIENT_CERT_AUTH="true" ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" EOF
6.创建ETCD系统服务
[root@k8s-master ssl]# cat > /etc/systemd/system/etcd.service <<EOF [Unit] Description=Etcd Server After=network.target [Service] Type=simple WorkingDirectory=/var/lib/etcd EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf # set GOMAXPROCS to number of processors ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /opt/kubernetes/bin/etcd" Type=notify [Install] WantedBy=multi-user.target EOF
7.文件分发到两个node节点中:
[root@k8s-master ~]# scp /opt/kubernetes/cfg/etcd.conf 10.200.3.106:/opt/kubernetes/cfg/ [root@k8s-master ~]# scp /etc/systemd/system/etcd.service 10.200.3.106:/etc/systemd/system/ [root@k8s-master ~]# scp /opt/kubernetes/cfg/etcd.conf 10.200.3.107:/opt/kubernetes/cfg/ [root@k8s-master ~]# scp /etc/systemd/system/etcd.service 10.200.3.107:/etc/systemd/system/
8.修改node节点etcd.conf文件,ETCD_NAME改为本机的hostname.ETCD...URLS改为本机的ip地址
在k8s-node-1 修改etcd.conf文件
[root@k8s-node-1 ~]# cat >/opt/kubernetes/cfg/etcd.conf <<EOF #[member] ETCD_NAME="k8s-node-1" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_SNAPSHOT_COUNTER="10000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" ETCD_LISTEN_PEER_URLS="https://10.200.3.106:2380" ETCD_LISTEN_CLIENT_URLS="https://10.200.3.106:2379,https://127.0.0.1:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" #ETCD_CORS="" #[cluster] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.106:2380" # if you use different ETCD_NAME (e.g. test), # set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380" ETCD_INITIAL_CLUSTER_STATE="new" ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster" ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.106:2379" #[security] CLIENT_CERT_AUTH="true" ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" PEER_CLIENT_CERT_AUTH="true" ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" EOF
在k8s-node-2节点 修改etcd.conf文件
[root@k8s-node-2 ~]# cat >/opt/kubernetes/cfg/etcd.conf <<EOF #[member] ETCD_NAME="k8s-node-2" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" #ETCD_SNAPSHOT_COUNTER="10000" #ETCD_HEARTBEAT_INTERVAL="100" #ETCD_ELECTION_TIMEOUT="1000" ETCD_LISTEN_PEER_URLS="https://10.200.3.107:2380" ETCD_LISTEN_CLIENT_URLS="https://10.200.3.107:2379,https://127.0.0.1:2379" #ETCD_MAX_SNAPSHOTS="5" #ETCD_MAX_WALS="5" #ETCD_CORS="" #[cluster] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.107:2380" # if you use different ETCD_NAME (e.g. test), # set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380" ETCD_INITIAL_CLUSTER_STATE="new" ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster" ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.107:2379" #[security] CLIENT_CERT_AUTH="true" ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" PEER_CLIENT_CERT_AUTH="true" ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem" ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem" EOF
9.加载并启动系统服务(先启动node节点的ectd服务,然后在启动master端的etcd服务,避免timeout.):
[root@k8s-master ~]# systemctl daemon-reload [root@k8s-master ~]# systemctl enable etcd [root@k8s-master ~]# systemctl start etcd [root@k8s-master ~]# systemctl status etcd
10.验证集群
[root@k8s-master ~]#etcdctl --endpoints=https://10.200.3.105:2379 \ --ca-file=/opt/kubernetes/ssl/ca.pem \ --cert-file=/opt/kubernetes/ssl/etcd.pem \ --key-file=/opt/kubernetes/ssl/etcd-key.pem cluster-health
输出结果:
member ccbb1e9d1fcf5b1 is healthy: got healthy result from https://10.200.3.105:2379 member 69e75d4dd1a9a289 is healthy: got healthy result from https://10.200.3.106:2379 member fe4515ae4c34c4e2 is healthy: got healthy result from https://10.200.3.107:2379 cluster is healthy
使用etcdctl member list 来查看集群中的成员列表
#etcdctl --endpoints=https://10.200.3.105:2379 \ --ca-file=/opt/kubernetes/ssl/ca.pem \ --cert-file=/opt/kubernetes/ssl/etcd.pem \ --key-file=/opt/kubernetes/ssl/etcd-key.pem member list
输出结果:
4c650c413eacdd52: name=k8s-master peerURLs=https://10.200.3.105:2380 clientURLs=https://10.200.3.105:2379 isLeader=false bc2b7558b8439c74: name=k8s-node-2 peerURLs=https://10.200.3.107:2380 clientURLs=https://10.200.3.107:2379 isLeader=false d71aa8654024396a: name=k8s-node-1 peerURLs=https://10.200.3.106:2380 clientURLs=https://10.200.3.106:2379 isLeader=true
至此etcd集群搭建完成!!!