kubernetes高可用集群安装

1、相关资源导航
https://blog.csdn.net/zyj81092211/article/details/123068347

2、环境介绍
操作系统：centos 7.9
kubernetes版本：v1.23
kubelet kubeadm kubectl版本：1.23.4
containerd版本：1.4.13
master配置：2核心 4G内存 80G硬盘
worker配置：16核心 16G内存 500G硬盘
ip地址规划：
k8s-master01:10.99.24.31
k8s-master02:10.99.24.32
k8s-master03:10.99.24.33
k8s-worker01:10.99.24.34
k8s-worker02:10.99.24.35
k8s-worker03:10.99.24.36
k8s-worker04:10.99.24.37
apiserver-lb01：10.99.24.28
apiserver-lb01：10.99.24.29
apiserver VIP:10.99.24.30

3、关闭防火墙和selinux（3-16步，在所有k8s节点上执行）

4、设置主机名（设置所有主机）

hostnamectl set-hostname k8s-master01

其他节点类似

5、设置hosts文件主机映射

10.99.24.31     k8s-master01    
10.99.24.32     k8s-master02    
10.99.24.33     k8s-master03    
10.99.24.34     k8s-worker01
10.99.24.35     k8s-worker02
10.99.24.36     k8s-worker03
10.99.24.37     k8s-worker04
10.99.24.30     apiserver-vip

6、安装工具包

yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git

7、关闭交换分区

swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

8、调整内核参数

cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963 fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

cp kubernetes.conf /etc/sysctl.d/kubernetes.conf

sysctl -p /etc/sysctl.d/kubernetes.conf

9、关闭不必要的程序

systemctl stop postfix && systemctl disable postfix

10、系统日志设置

mkdir /var/log/journal   # 持久化保存日志的录
mkdir /etc/systemd/journald.conf.d

cat > /etc/systemd/journald.conf.d/99-prophet.conf<<EOF
[Journal]
#  持久化保存到磁盘
 Storage=persistent
#  压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
 #  最大占用空间 10G
 SystemMaxUse=10G
 #  单日志文件最大 200M
SystemMaxFileSize=200M
 #  日志保存时间 2 周 
MaxRetentionSec=2week
 #  不将日志转发到 syslog
 ForwardToSyslog=no
EOF

systemctl restart systemd-journald

11、升级系统内核
https://blog.csdn.net/zyj81092211/article/details/123350787

12、设置使用ipvs

modprobe br_netfilter

cat > /etc/sysconfig/modules/ipvs.modules <<EOF 
#!/bin/bash
modprobe -- ip_vs 
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr 
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack

13、安装containerd

cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system

设置源

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

安装containerd

yum install -y containerd.io

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

sed -i "s#k8s.gcr.io#registry.aliyuncs.com/k8sxio#g"  /etc/containerd/config.toml
sed -i '/containerd.runtimes.runc.options/a\ \ \ \ \ \ \ \ \ \ \ \ SystemdCgroup = true' /etc/containerd/config.toml
sed -i "s#https://registry-1.docker.io#https://registry.cn-hangzhou.aliyuncs.com#g"  /etc/containerd/config.toml

启动

systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd

14、安装nfs-utils（挂载nfs网络存储用）

yum install -y nfs-utils

15、配置Kubernetes软件源

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

16、安装kubelet kubeadm kubectl

yum install -y kubelet-1.23.4 kubeadm-1.23.4 kubectl-1.23.4

crictl config runtime-endpoint /run/containerd/containerd.sock

启动

systemctl daemon-reload
systemctl enable kubelet && systemctl start kubelet

containerd --version
kubelet --version

17、设置apiserver 负载均衡器（在apiserver 负载均衡节点上）
安装docker

curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun

启动docker

systemctl start docker
systemctl enable docker

创建脚本目录

mkdir -p /data/lb

创建HaProxy启动脚本

cat > /data/lb/start-haproxy.sh << "EOF"
#!/bin/bash
MasterIP1=10.99.24.31
MasterIP2=10.99.24.32
MasterIP3=10.99.24.33
MasterPort=6443

docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
        -e MasterIP1=$MasterIP1 \
        -e MasterIP2=$MasterIP2 \
        -e MasterIP3=$MasterIP3 \
        -e MasterPort=$MasterPort \
        wise2c/haproxy-k8s
EOF

keepalived启动脚本

cat > /data/lb/start-keepalived.sh << "EOF"
#!/bin/bash
VIRTUAL_IP=10.99.24.30
INTERFACE=ens192
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18

docker run -itd --restart=always --name=Keepalived-K8S \
        --net=host --cap-add=NET_ADMIN \
        -e VIRTUAL_IP=$VIRTUAL_IP \
        -e INTERFACE=$INTERFACE \
        -e CHECK_PORT=$CHECK_PORT \
        -e RID=$RID \
        -e VRID=$VRID \
        -e NETMASK_BIT=$NETMASK_BIT \
        -e MCAST_GROUP=$MCAST_GROUP \
        wise2c/keepalived-k8s
EOF

脚本添加执行权限

chmod +x /data/lb/*

启动haproxy脚本

/data/lb/start-haproxy.sh

启动keepalived脚本

/data/lb/start-keepalived.sh

18、初始化master节点
获取默认配置文件

kubeadm config print init-defaults > kubeadm-config.yaml

官方配置文件解析
https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta3/

初始化文件kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: "v1.23.4"
imageRepository: registry.aliyuncs.com/google_containers
controlPlaneEndpoint: "10.99.24.30:6444"
networking:
  serviceSubnet: "172.199.0.0/16"
  podSubnet: "172.200.0.0/16"
  dnsDomain: "cluster.local"
clusterName: "GB-cluster"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

注意：启用ipvs说明
https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/README.md

拉取镜像

kubeadm config images pull --config=kubeadm-config.yaml

初始化master节点

kubeadm init --config=kubeadm-config.yaml --upload-certs

加入其他两个主节点

kubeadm join 10.99.24.30:6444 --token uq57pt.p0bn7zlg3mmxird8 \
	--discovery-token-ca-cert-hash sha256:cdb45364d603df283013fd8198260a0e763930d1484257014e2558fa54b2f4cc \
	--control-plane --certificate-key 70919992d633796ed501edf3d090a76284ee05854b96f3957f833ea809e54b4b

查看现在正在使用的初始化配置文件

kubectl -n kube-system get cm kubeadm-config -o yaml

为master节点设置环境变量

echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /etc/profile && source /etc/profile

19、安装网络插件Calico

kubectl create -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yaml
wget https://projectcalico.docs.tigera.io/manifests/custom-resources.yaml --no-check-certificate
sed -i "s#192.168.0.0/16#172.200.0.0/16#" custom-resources.yaml
kubectl create -f custom-resources.yaml

20、加入worker节点
生成票据

kubeadm token create --print-join-command

kubeadm join 10.99.24.30:6444 --token 2kx25k.j3dxrgdyo0gczrqb --discovery-token-ca-cert-hash sha256:cdb45364d603df283013fd8198260a0e763930d1484257014e2558fa54b2f4cc

21、检查节点和pod是否运行正常

kubectl get node -o wide

kubectl get pods --all-namespaces -o wide