1、相关资源导航
https://blog.csdn.net/zyj81092211/article/details/123068347
2、环境介绍
操作系统:centos 7.9
kubernetes版本:v1.23
kubelet kubeadm kubectl版本:1.23.4
containerd版本:1.4.13
master配置:2核心 4G内存 80G硬盘
worker配置:16核心 16G内存 500G硬盘
ip地址规划:
k8s-master01:10.99.24.31
k8s-master02:10.99.24.32
k8s-master03:10.99.24.33
k8s-worker01:10.99.24.34
k8s-worker02:10.99.24.35
k8s-worker03:10.99.24.36
k8s-worker04:10.99.24.37
apiserver-lb01:10.99.24.28
apiserver-lb01:10.99.24.29
apiserver VIP:10.99.24.30
3、关闭防火墙和selinux(3-16步,在所有k8s节点上执行)
4、设置主机名(设置所有主机)
hostnamectl set-hostname k8s-master01
其他节点类似
5、设置hosts文件主机映射
10.99.24.31 k8s-master01
10.99.24.32 k8s-master02
10.99.24.33 k8s-master03
10.99.24.34 k8s-worker01
10.99.24.35 k8s-worker02
10.99.24.36 k8s-worker03
10.99.24.37 k8s-worker04
10.99.24.30 apiserver-vip
6、安装工具包
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
7、关闭交换分区
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
8、调整内核参数
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间,只有当系统 OOM 时才允许使它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963 fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
9、关闭不必要的程序
systemctl stop postfix && systemctl disable postfix
10、系统日志设置
mkdir /var/log/journal # 持久化保存日志的录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf<<EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent
# 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空间 10G
SystemMaxUse=10G
# 单日志文件最大 200M
SystemMaxFileSize=200M
# 日志保存时间 2 周
MaxRetentionSec=2week
# 不将日志转发到 syslog
ForwardToSyslog=no
EOF
systemctl restart systemd-journald
11、升级系统内核
https://blog.csdn.net/zyj81092211/article/details/123350787
12、设置使用ipvs
modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack
13、安装containerd
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
设置源
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
安装containerd
yum install -y containerd.io
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
sed -i "s#k8s.gcr.io#registry.aliyuncs.com/k8sxio#g" /etc/containerd/config.toml
sed -i '/containerd.runtimes.runc.options/a\ \ \ \ \ \ \ \ \ \ \ \ SystemdCgroup = true' /etc/containerd/config.toml
sed -i "s#https://registry-1.docker.io#https://registry.cn-hangzhou.aliyuncs.com#g" /etc/containerd/config.toml
启动
systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd
14、安装nfs-utils(挂载nfs网络存储用)
yum install -y nfs-utils
15、配置Kubernetes软件源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
16、安装kubelet kubeadm kubectl
yum install -y kubelet-1.23.4 kubeadm-1.23.4 kubectl-1.23.4
crictl config runtime-endpoint /run/containerd/containerd.sock
启动
systemctl daemon-reload
systemctl enable kubelet && systemctl start kubelet
containerd --version
kubelet --version
17、设置apiserver 负载均衡器(在apiserver 负载均衡节点上)
安装docker
curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
启动docker
systemctl start docker
systemctl enable docker
创建脚本目录
mkdir -p /data/lb
创建HaProxy启动脚本
cat > /data/lb/start-haproxy.sh << "EOF"
#!/bin/bash
MasterIP1=10.99.24.31
MasterIP2=10.99.24.32
MasterIP3=10.99.24.33
MasterPort=6443
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
-e MasterIP1=$MasterIP1 \
-e MasterIP2=$MasterIP2 \
-e MasterIP3=$MasterIP3 \
-e MasterPort=$MasterPort \
wise2c/haproxy-k8s
EOF
keepalived启动脚本
cat > /data/lb/start-keepalived.sh << "EOF"
#!/bin/bash
VIRTUAL_IP=10.99.24.30
INTERFACE=ens192
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s
EOF
脚本添加执行权限
chmod +x /data/lb/*
启动haproxy脚本
/data/lb/start-haproxy.sh
启动keepalived脚本
/data/lb/start-keepalived.sh
18、初始化master节点
获取默认配置文件
kubeadm config print init-defaults > kubeadm-config.yaml
官方配置文件解析
https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta3/
初始化文件kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: "v1.23.4"
imageRepository: registry.aliyuncs.com/google_containers
controlPlaneEndpoint: "10.99.24.30:6444"
networking:
serviceSubnet: "172.199.0.0/16"
podSubnet: "172.200.0.0/16"
dnsDomain: "cluster.local"
clusterName: "GB-cluster"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
注意:启用ipvs说明
https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/README.md
拉取镜像
kubeadm config images pull --config=kubeadm-config.yaml
初始化master节点
kubeadm init --config=kubeadm-config.yaml --upload-certs
加入其他两个主节点
kubeadm join 10.99.24.30:6444 --token uq57pt.p0bn7zlg3mmxird8 \
--discovery-token-ca-cert-hash sha256:cdb45364d603df283013fd8198260a0e763930d1484257014e2558fa54b2f4cc \
--control-plane --certificate-key 70919992d633796ed501edf3d090a76284ee05854b96f3957f833ea809e54b4b
查看现在正在使用的初始化配置文件
kubectl -n kube-system get cm kubeadm-config -o yaml
为master节点设置环境变量
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /etc/profile && source /etc/profile
19、安装网络插件Calico
kubectl create -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yaml
wget https://projectcalico.docs.tigera.io/manifests/custom-resources.yaml --no-check-certificate
sed -i "s#192.168.0.0/16#172.200.0.0/16#" custom-resources.yaml
kubectl create -f custom-resources.yaml
20、加入worker节点
生成票据
kubeadm token create --print-join-command
kubeadm join 10.99.24.30:6444 --token 2kx25k.j3dxrgdyo0gczrqb --discovery-token-ca-cert-hash sha256:cdb45364d603df283013fd8198260a0e763930d1484257014e2558fa54b2f4cc
21、检查节点和pod是否运行正常
kubectl get node -o wide
kubectl get pods --all-namespaces -o wide