![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/e7171f409db5ab144a5327937a5cd80b.png#pic_center)
集群结构
192.168.0.107 master01
192.168.0.108 master02
192.168.0.109 master03
192.168.0.110 node01
192.168.0.111 node02
192.168.0.236
10.96.0.0/12
172.16.0.0/12
CentOS Linux release 7.9.2009 (Core)
Linux k8s-master02 4.19.12-1.el7.elrepo.x86_64
目录结构
/etc/kubernetes/
环境配置
192.168.0.107 master01
192.168.0.108 master02
192.168.0.109 master03
192.168.0.236 master-lb
192.168.0.110 node01
192.168.0.111 node02
cd /etc/yum.repos.d/
rm -rf *
wget http://mirrors.aliyun.com/repo/Centos-7.repo
wget http://mirrors.aliyun.com/repo/epel-7.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/Centos-7.repo
yum clean all && yum makecache
yum -y install wget vim git telnet jq psmisc net-tools yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum -y install ntpdate
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
vim /etc/security/limits.conf
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
- 关闭firewalld 、NetworkManager
systemctl disable --now firewalld
systemctl disable --now NetworkManager
getenforce
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
ssh-keygen -t rsa
for i in master02 master03 node01 node02;
do
ssh-copy-id -i /root/.ssh/id_rsa.pub $i;
done
yum -y update --exclude=kernel*
cd /root
wget http://193.49.22.109/ elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
for i in master02 master03;
do
scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm $i:/root/ ;
done
cd /root
yum -y localinstall kernel-ml*
grub2-set-default 0
grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
reboot
grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64
uname -a
Linux k8s-master02 4.19.12-1.el7.elrepo.x86_64
yum -y install ipvsadm ipset sysstat conntrack libseccomp
vim /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
systemctl enable --now systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_ftp 16384 0
nf_nat 32768 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 151552 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 143360 2 nf_nat,ip_vs
nf_defrag_ipv6 20480 1 nf_conntrack
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 4 nf_conntrack,nf_nat,xfs,ip_vs
cat > /etc/sysctl.d/k8s.conf <<EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system
reboot
安装组件
yum -y install docker-ce-19.03.*
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl daemon-reload && systemctl enable --now docker
docker info | grep Driver
Storage Driver: overlay2
Logging Driver: json-file
Cgroup Driver: systemd
wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz
tar -zxvf etcd-v3.4.13-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.4.13-linux-amd64/etcd{
,ctl}
etcdctl version
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG
/usr/local/bin/kube{
let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
tar -zxvf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{
let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
kubelet --version
MasterNodes='master02 master03'
WorkNodes='node01 node02'
for NODE in $MasterNodes;
do
echo $NODE;
scp /usr/local/bin/kube{
let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/;
scp /usr/local/bin/etcd* $NODE:/usr/local/bin/;
done
for NODE in $WorkNodes;
do
scp /usr/local/bin/kube{
let,-proxy} $NODE:/usr/local/bin/ ;
done
mkdir -p /opt/cni/bin
cd /root
git clone https://github.com/dotbalo/k8s-ha-install.git
cd /root/k8s-ha-install
git branch -a
* master
remotes/origin/HEAD -> origin/master
remotes/origin/manual-installation
remotes/origin/manual-installation-v1.16.x
remotes/origin/manual-installation-v1.17.x
remotes/origin/manual-installation-v1.18.x
remotes/origin/manual-installation-v1.19.x
remotes/origin/manual-installation-v1.20.x
remotes/origin/manual-installation-v1.20.x-csi-hostpath
remotes/origin/master
git checkout manual-installation-v1.20.x
etcd证书
wget -O /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget -O /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmod +x /usr/local/bin/cfssl
chmod +x /usr/local/bin/cfssljson
cfssl print-defaults config > config.json
cfssl print-defaults csr > csr.json
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "876000h"
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF
mkdir -p /etc/etcd/ssl
cd /root/k8s-ha-install/pki
cfssl gencert \
-initca etcd-ca-csr.json | cfssljson \
-bare /etc/etcd/ssl/etcd-ca
2021/04/05 18:14:26 [INFO] generating a new CA key and certificate from CSR
2021/04/05 18:14:26 [INFO] generate received request
2021/04/05 18:14:26 [INFO] received CSR
2021/04/05 18:14:26 [INFO] generating key: rsa-2048
2021/04/05 18:14:26 [INFO] encoded CSR
2021/04/05 18:14:26 [INFO] signed certificate with serial number 412473371421533418369334251703332009328607963759
cfssl gencert \
-ca=/etc/etcd/ssl/etcd-ca.pem \
-ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
-config=ca-config.json \
-hostname=127.0.0.1,master01,master02,master03,192.168.0.107,192.168.0.108,192.168.0.109 \
-profile=kubernetes etcd-csr.json | cfssljson
-bare /etc/etcd/ssl/etcd
2021/03/28 16:25:13 [INFO] generate received request
2021/03/28 16:25:13 [INFO] received CSR
2021/03/28 16:25:13 [INFO] generating key: rsa-2048
2021/03/28 16:25:13 [INFO] encoded CSR
2021/03/28 16:25:13 [INFO] signed certificate with serial number 456951054614724417062321915302964486528346522769
MasterNodes='192.168.0.108 192.168.0.109'
for NODE in $MasterNodes; do
ssh $NODE "mkdir -p /etc/etcd/ssl"
for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do
scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}
done
done
k8s证书
mkdir -p /etc/kubernetes/pki
cd /root/k8s-ha-install/pki
cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
2021/03/28 17:09:53 [INFO] generating a new CA key and certificate from CSR
2021/03/28 17:09:53 [INFO] generate received request
2021/03/28 17:09:53 [INFO] received CSR
2021