四、安装k8s(control plane和work节点)
注意:如果以前已经安装kubectl ,kubelet和kubeadm 需要先行卸载
4.1 安装etcd(分别在三台master服务器上安装etcd)
(所有etcd节点k8smaster01、k8smaster02、k8smaster03)
mkdir /opt/kubernetes/{bin,config,ssl}
wget https://github.com/etcd-io/etcd/releases/download/v3.4.4/etcd-v3.4.4-linux-amd64.tar.gz
tar -xvzf etcd-v3.4.4-linux-amd64.tar.gz
cp etcd etcdctl /opt/kubernetes/bin
#下载TLS证书生产工具
curl -s -L -o /opt/kubernetes/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl -s -L -o /opt/kubernetes/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmod +x /opt/kubernetes/bin/{cfssl,cfssljson}
临时:
export PATH=$PATH:/opt/kubernetes/bin/
永久:
vi ~/.bash_profile
PATH=$PATH:/opt/kubernetes/bin/
#source ~/.bash_profile
cd /opt/kubernetes/ssl
cat >ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"kubernetes": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"etcd": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat << EOF > ca-csr.json
{
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
#执行cfssl命令,生产CA证书
cfssl gencert --initca ca-csr.json | cfssljson --bare ca
#生产3个文件ca.csr,ca-key.pem,ca.pem,如果需要重新生成证书,这三个文件都必须全部删除,再重新生成
ll ca*
[root@k8smaster01 ssl]# ll ca*
-rw-r--r-- 1 root root 640 Mar 13 23:16 ca-config.json
-rw-r--r-- 1 root root 972 Mar 13 23:16 ca.csr
-rw-r--r-- 1 root root 240 Mar 13 23:16 ca-csr.json
-rw------- 1 root root 1675 Mar 13 23:16 ca-key.pem
-rw-r--r-- 1 root root 1302 Mar 13 23:16 ca.pem
cd /opt/kubernetes/ssl
cat << EOF > etcd-csr.json
{
"CN": "etcd",
"hosts": [
"10.111.69.240",
"10.111.83.165",
"10.111.127.129"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
#执行cfssl命令生成etcd 证书签名,这需要用到前面生成的CA证书,以及ca-config.json
cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem -ca-key=/opt/kubernetes/ssl/ca-key.pem -config=/opt/kubernetes/ssl/ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
#生产3个文件etcd.csr,etcd-key.pem,etcd.pem,如果需要重新生成证书,这三个文件都必须全部删除,再重新生成
[root@k8smaster01 ssl]# ll etcd*
-rw-r--r-- 1 root root 1054 Mar 13 23:10 etcd.csr
-rw-r--r-- 1 root root 283 Mar 13 23:08 etcd-csr.json
-rw------- 1 root root 1675 Mar 13 23:10 etcd-key.pem
-rw-r--r-- 1 root root 1395 Mar 13 23:10 etcd.pem
注意:(为了和flannel 进行兼容,必须 设置ETCD_ENABLE_V2="true")否则启动flannel时候会报错Error: client: response is invalid json. The endpoint is probably not valid etcd cluster endpoint
k8smaster01
cat > /opt/kubernetes/config/etcd << EOF
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.111.69.240:2380" # 本节点IP地址,不能用DNS name
ETCD_LISTEN_CLIENT_URLS="https://10.111.69.240:2379" # 本节点IP地址,不能用DNS name
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.111.69.240:2380" # 本节点IP地址,不能用DNS name
ETCD_ADVERTISE_CLIENT_URLS="https://10.111.69.240:2379" # 本节点IP地址,不能用DNS name
ETCD_INITIAL_CLUSTER="etcd01=https://10.111.69.240:2380,etcd02=https://10.111.83.165:2380,etcd03=https://10.111.127.129:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_ENABLE_V2="true"
EOF
k8smaster02
cat > /opt/kubernetes/config/etcd << EOF
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.111.83.165:2380" # 本节点IP地址,不能用DNS name
ETCD_LISTEN_CLIENT_URLS="https://10.111.83.165:2379" # 本节点IP地址,不能用DNS name
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.111.83.165:2380" # 本节点IP地址,不能用DNS name
ETCD_ADVERTISE_CLIENT_URLS="https://10.111.83.165:2379" # 本节点IP地址,不能用DNS name
ETCD_INITIAL_CLUSTER="etcd01=https://10.111.69.240:2380,etcd02=https://10.111.83.165:2380,etcd03=https://10.111.127.129:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_ENABLE_V2="true"
EOF
k8smaster03
cat > /opt/kubernetes/config/etcd << EOF
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.111.127.129:2380" # 本节点IP地址,不能用DNS name
ETCD_LISTEN_CLIENT_URLS="https://10.111.127.129:2379" # 本节点IP地址,不能用DNS name
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.111.127.129:2380" # 本节点IP地址,不能用DNS name
ETCD_ADVERTISE_CLIENT_URLS="https://10.111.127.129:2379" # 本节点IP地址,不能用DNS name
ETCD_INITIAL_CLUSTER="etcd01=https://10.111.69.240:2380,etcd02=https://10.111.83.165:2380,etcd03=https://10.111.127.129:2380"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_ENABLE_V2="true"
EOF
4.1.5 创建 systemd 管理etcd.service
cat > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=-/opt/kubernetes/config/etcd
ExecStart=/opt/kubernetes/bin/etcd \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/opt/kubernetes/ssl/etcd.pem \
--peer-key-file=/opt/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
注意:ETCD3.4版本ETCDCTL_API=3 etcdctl 和 etcd --enable-v2=false 成为了默认配置,如要使用v2版本,执行etcdctl时候需要设置ETCDCTL_API环境变量,例如:ETCDCTL_API=2 etcdctl
ETCD3.4版本会自动读取环境变量的参数,所以EnvironmentFile文件中有的参数,不需要再次在ExecStart启动参数中添加,二选一,如同时配置,会触发以下类似报错“etcd: conflicting environment variable "ETCD_NAME" is shadowed by corresponding command-line flag (either unset environment variable or disable flag)”
flannel操作etcd使用的是v2的API,而kubernetes操作etcd使用的v3的API
4.1.6 同步证书、etcd配置文件和etcd.service文件
for i in {k8smaster02,k8smaster03}
do
scp /opt/kubernetes/ssl/* $i://opt/kubernetes/ssl/
scp /opt/kubernetes/config/etcd $i://opt/kubernetes/config/
scp /usr/lib/systemd/system/etcd.service $i://usr/lib/systemd/system/
done
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
[root@k8smaster03 ssl]# systemctl status etcd
â— etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2020-03-13 23:19:30 EDT; 46min ago
Main PID: 25190 (etcd)
CGroup: /system.slice/etcd.service
└─25190 /opt/kubernetes/bin/etcd --cert-file=/opt/kubernetes/ssl/etcd.pem --key-file=/opt/kubernetes/ssl/etcd-k...
Mar 13 23:19:30 k8smaster03 etcd[25190]: raft2020/03/13 23:19:30 INFO: 77c428cbe41236c2 [logterm: 1222, index: 85, v...m 1452
Mar 13 23:19:30 k8smaster03 etcd[25190]: raft2020/03/13 23:19:30 INFO: raft.node: 77c428cbe41236c2 elected leader fa...m 1452
Mar 13 23:19:30 k8smaster03 etcd[25190]: set the initial cluster version to 3.0
Mar 13 23:19:30 k8smaster03 etcd[25190]: enabled capabilities for version 3.0
Mar 13 23:19:30 k8smaster03 etcd[25190]: ready to serve client requests
Mar 13 23:19:30 k8smaster03 etcd[25190]: published {Name:etcd03 ClientURLs:[https://10.111.127.129:2379]} to cluster...f5d617
Mar 13 23:19:30 k8smaster03 systemd[1]: Started Etcd Server.
Mar 13 23:19:30 k8smaster03 etcd[25190]: serving client requests on 10.111.127.129:2379
Mar 13 23:19:30 k8smaster03 etcd[25190]: updated the cluster version from 3.0 to 3.4
Mar 13 23:19:30 k8smaster03 etcd[25190]: enabled capabilities for version 3.4
Hint: Some lines were ellipsized, use -l to show in full.
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
[root@k8smaster03 ssl]# systemctl status etcd
â— etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2020-03-13 23:19:30 EDT; 46min ago
Main PID: 25190 (etcd)
CGroup: /system.slice/etcd.service
└─25190 /opt/kubernetes/bin/etcd --cert-file=/opt/kubernetes/ssl/etcd.pem --key-file=/opt/kubernetes/ssl/etcd-k...
Mar 13 23:19:30 k8smaster03 etcd[25190]: raft2020/03/13 23:19:30 INFO: 77c428cbe41236c2 [logterm: 1222, index: 85, v...m 1452
Mar 13 23:19:30 k8smaster03 etcd[25190]: raft2020/03/13 23:19:30 INFO: raft.node: 77c428cbe41236c2 elected leader fa...m 1452
Mar 13 23:19:30 k8smaster03 etcd[25190]: set the initial cluster version to 3.0
Mar 13 23:19:30 k8smaster03 etcd[25190]: enabled capabilities for version 3.0
Mar 13 23:19:30 k8smaster03 etcd[25190]: ready to serve client requests
Mar 13 23:19:30 k8smaster03 etcd[25190]: published {Name:etcd03 ClientURLs:[https://10.111.127.129:2379]} to cluster...f5d617
Mar 13 23:19:30 k8smaster03 systemd[1]: Started Etcd Server.
Mar 13 23:19:30 k8smaster03 etcd[25190]: serving client requests on 10.111.127.129:2379
Mar 13 23:19:30 k8smaster03 etcd[25190]: updated the cluster version from 3.0 to 3.4
Mar 13 23:19:30 k8smaster03 etcd[25190]: enabled capabilities for version 3.4
Hint: Some lines were ellipsized, use -l to show in full.