从入门到放弃之生产环境基于Kubernetes V1.21.10搭建高可用集群

1.资源规划

名称	系统	配置	IP	组件
k8s-master01	CentOS 7.9	4核8G 500G存储	172.16.97.27	kube-apiserver <br>kube-controller-manager <br>kube-scheduler <br>etcd
k8s-master02	CentOS 7.9	4核8G 500G存储	172.16.97.28	kube-apiserver <br>kube-controller-manager <br>kube-scheduler <br>etcd
k8s-master03	CentOS 7.9	4核8G 500G存储	172.16.97.29	kube-apiserver <br>kube-controller-manager <br>kube-scheduler <br>etcd
k8s-node01	CentOS 7.9	8核16G 500G存储	172.16.97.30	kubelet <br>kube-proxy <br>docker
k8s-node02	CentOS 7.9	8核16G 500G存储	172.16.97.31	kubelet <br>kube-proxy <br>docker
k8s-node03	CentOS 7.9	8核16G 500G存储	172.16.97.32	kubelet <br>kube-proxy <br>docker
HA01	CentOS 7.9	4核8G 60G存储	172.16.97.33 <br>172.16.97.35(vip)	haproxy、keepalived
HA02	CentOS 7.9	4核8G 60G存储	172.16.97.34	haproxy、keepalived

2.系统初始化

1.主机名设置

# hostnamectl set-hostname xxx

2.主机与IP地址解析

cat >> /etc/hosts << EOF
172.16.97.33 ha1
172.16.97.34 ha2
172.16.97.27 k8s-master01
172.16.97.28 k8s-master02
172.16.97.29 k8s-master03
172.16.97.30 k8s-node01
172.16.97.31 k8s-node02
172.16.97.32 k8s-node03
EOF

3.关闭防火墙，生产环境不建议关闭防火墙

# systemctl stop firewalld
# systemctl disable firewalld

4.关闭selinx

# sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
# sestatus

5.关闭wasp

# sed -ri 's/.*swap.*/#&/' /etc/fstab
# sysctl -p
# free -m 确认是否关闭

6.同步时间

# yum -y install ntpdate
# ntpdate ntp1.aliyun.com

7.主机系统优化

cat <<EOF >> /etc/security/limits.conf
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF

1. 升级操作系统内核

所有主机均需要操作。

导入elrepo gpg key
# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
安装elrepo YUM源仓库
# yum -y install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
安装kernel-ml版本，ml为长期稳定版本，lt为长期维护版本
# yum --enablerepo="elrepo-kernel" -y install kernel-ml.x86_64
设置grub2默认引导为0
# grub2-set-default 0
重新生成grub2引导文件
# grub2-mkconfig -o /boot/grub2/grub.cfg
更新后，需要重启，使用升级的内核生效。
# reboot
重启后，需要验证内核是否为更新对应的版本
# uname -r

9.Linux内核优化

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 131072
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

sysctl --system

10.ipvs管理工具安装及模块加载

为集群节点安装，负载均衡节点不用安装

# yum -y install ipvsadm ipset sysstat conntrack libseccomp
所有节点配置ipvs模块，在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack， 4.18以下使用nf_conntrack_ipv4即可： 
 
创建 /etc/modules-load.d/ipvs.conf 并加入以下内容： 
cat >/etc/modules-load.d/ipvs.conf <<EOF 
ip_vs 
ip_vs_lc 
ip_vs_wlc 
ip_vs_rr 
ip_vs_wrr 
ip_vs_lblc 
ip_vs_lblcr 
ip_vs_dh 
ip_vs_sh 
ip_vs_fo 
ip_vs_nq 
ip_vs_sed 
ip_vs_ftp 
ip_vs_sh 
nf_conntrack 
ip_tables 
ip_set 
xt_set 
ipt_set 
ipt_rpfilter 
ipt_REJECT 
ipip 
EOF

设置为开机启动
# systemctl enable --now systemd-modules-load.service

所有节点配置完内核后，重启服务器，保证重启后内核依旧加载
# reboot -h now
重启后查看ipvs模块加载情况：
# lsmod | grep --color=auto -e ip_vs -e nf_conntrack

11.依赖安装（负载均衡节点不安装）

# yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git lrzsz -y

3.etcd集群搭建

在k8s-master01上操作。

1.创建工作目录

# mkdir -p /data/k8s-work

2.获取cfssl工具

# cd /data/k8s-work
# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64

# chmod +x cfssl*

# mv cfssl_linux-amd64 /usr/local/bin/cfssl
# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

# cfssl version

3.创建CA证书

3.1.配置ca证书请求文件

cat > ca-csr.json <<"EOF"
{
  "CN": "kubernetes",
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "beijing",
      "L": "beijing",
      "O": "kubemsb",
      "OU": "CN"
    }
  ],
  "ca": {
          "expiry": "87600h"
  }
}
EOF

3.2.创建ca证书

# cfssl gencert -initca ca-csr.json | cfssljson -bare ca

3.3.配置ca证书策略

# cfssl print-defaults config > ca-config.json
cat > ca-config.json <<"EOF"
{
  "signing": {
      "default": {
          "expiry": "87600h"
        },
      "profiles": {
          "kubernetes": {
              "usages": [
                  "signing",
                  "key encipherment",
                  "server auth",
                  "client auth"
              ],
              "expiry": "87600h"
          }
      }
  }
}
EOF

4. 创建etcd证书

4.1.配置etcd请求文件

cat > etcd-csr.json <<"EOF"
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "172.16.97.27",
    "172.16.97.28",
    "172.16.97.29"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "CN",
    "ST": "Beijing",
    "L": "Beijing",
    "O": "kubemsb",
    "OU": "CN"
  }]
}
EOF

4.2.生成etcd证书

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson  -bare etcd

5.部署etcd集群

5.1.下载etcd软件包

# wget https://github.com/etcd-io/etcd/releases/download/v3.5.2/etcd-v3.5.2-linux-amd64.tar.gz

5.2.安装etcd软件

# tar -xvf etcd-v3.5.2-linux-amd64.tar.gz
# cp -p etcd-v3.5.2-linux-amd64/etcd* /usr/local/bin/

5.3.分发etcd软件

# scp etcd-v3.5.2-linux-amd64/etcd* k8s-master02:/usr/local/bin/
# scp etcd-v3.5.2-linux-amd64/etcd* k8s-master03:/usr/local/bin/

5.4.创建配置文件

# mkdir /etc/etcd
k8s-master01：
cat >  /etc/etcd/etcd.conf <<"EOF"
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/data/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.97.27:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.97.27:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.97.27:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.97.27:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://172.16.97.27:2380,etcd02=https://172.16.97.28:2380,etcd03=https://172.16.97.29:2380" 
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

k8s-master02：
cat >  /etc/etcd/etcd.conf <<"EOF"
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/data/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.97.28:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.97.28:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.97.28:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.97.28:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://172.16.97.27:2380,etcd02=https://172.16.97.28:2380,etcd03=https://172.16.97.29:2380" 
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
k8s-master03：
cat >  /etc/etcd/etcd.conf <<"EOF"
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/data/etcd/default.etcd" 
ETCD_LISTEN_PEER_URLS="https://172.16.97.29:2380" 
ETCD_LISTEN_CLIENT_URLS="https://172.16.97.29:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.97.29:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.97.29:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://172.16.97.27:2380,etcd02=https://172.16.97.28:2380,etcd03=https://172.16.97.29:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" 
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

说明：
ETCD_NAME：节点名称，集群中唯一
ETCD_DATA_DIR：数据目录
ETCD_LISTEN_PEER_URLS：集群通信监听地址
ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
ETCD_INITIAL_ADVERTISE_PEER_URLS：集群通告地址
ETCD_A