kubernetesv1.23.5在线安装（kubeadm+contained）

一.基础环境

本次安装采用的是Kubeadm安装工具，安装版本是K8s 1.23.5，采用的系统为CentOS 7.9，内核版本为：5.17.4-1.el7.elrepo.x86_64，其中Master节点3台，Node节点2台。

基础环境优化（所有节点）

所有节点配置hosts

cat /etc/hosts
10.20.0.201 k8s-master01

10.20.0.202 k8s-master02

10.20.0.203 k8s-master03

10.20.0.200 k8s-master-lb # 如果不是高可用集群，该IP为Master01的IP

10.20.0.204 k8s-node01

10.20.205 k8s-node02

配置yum源

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo


yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo


cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

所有节点关闭防火墙、selinux、dnsmasq、swap。服务器配置如下

#节点关闭防火墙 dnsmasq NetworkManager
systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager

#禁用selinux
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

禁用swap
swapoff -a && sysctl -w vm.swappiness=0
#永久禁用
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
#sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
#临时禁用
setenforce 0
getenforce 

同步时间

#1.通过ntpdate同步时间
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y
ntpdate ntp1.aliyun.com

#2.通过chrony同步时间
yum install chrony -y
systemctl enable chronyd
systemctl start chronyd
chronyc sources  #时间同步

调整内核参数

#开启ipvs的内核参数
cat <<EOF> /etc/modules-load.d/k8s.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack_ipv4
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

systemctl enable --now systemd-modules-load.service

cat <<EOF> /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

# 应用 sysctl 参数而无需重新启动
sudo sysctl --system

#配置ulimit参数
vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

#配置k8s的内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system

所有节点安装ipvsadm

yum install ipvsadm ipset sysstat conntrack libseccomp -y

节点安装containerd，替换docker

# 查看最新版本
yum list containerd --showduplicates | sort -r
yum install containerd -y

containerd config default > /etc/containerd/config.toml
systemctl start containerd
systemctl enable containerd

# 修改cgroups为systemd
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#' /etc/containerd/config.toml
systemctl daemon-reload
systemctl restart containerd

# crictl 管理containerd
# 客户端地址： https://github.com/kubernetes-sigs/cri-tools/releases/
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.23.0/crictl-v1.23.0-linux-amd64.tar.gz
tar zxvf crictl-v1.23.0-linux-amd64.tar.gz -C /usr/local/bin

cat <<EOF> /etc/crictl.yaml 
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

# 验证是否可用
crictl pull nginx:alpine
crictl images
crictl rmi nginx:alpine

安装k8s组件

# 配置yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装
yum clean all
yum list kubeadm --showduplicates | sort -r
yum install -y kubelet-1.23.5-0 kubectl-1.23.5-0 kubeadm-1.23.5-0

kubelet指定runtime为containerd

cat <<EOF> /etc/sysconfig/kubelet
KUBELET_KUBEADM_ARGS="--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
EOF

# 启动kubelet
systemctl start kubelet
systemctl enable kubelet
————————————————

高可用组件安装

yum install keepalived haproxy -y

初始化集群

kubeadm config print init-defaults > kubeadm.yaml

#修改kubeadm.yaml
vim kubeadm.yaml
cat <<EOF> kubeadm.yaml 
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.20.0.200               # apiserver 节点内网IP
  bindPort: 6443
nodeRegistration:
  criSocket: /run/containerd/containerd.sock        # 修改为containerd
  imagePullPolicy: IfNotPresent
  name: master
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 10.20.0.200:6443
controllerManager: {}
dns:
  type: CoreDNS # dns类型 type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers # 修改这个镜像能下载
kind: ClusterConfiguration
kubernetesVersion: 1.23.5 # k8s版本
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16  
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs  # kube-proxy 模式
EOF


# kube-proxy 模式是 iptables，命令行
kubectl edit configmap kube-proxy -n kube-system修改

# 执行初始化
kubeadm init --config kubeadm.yaml

# 根据提示配置
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

#Alternatively, if you are the root user, you can run:

export KUBECONFIG=/etc/kubernetes/admin.conf
#也可以使用如下命令
cat <<EOF >> /root/.bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source /root/.bashrc

# 保留加入集群配置
$ kubeadm token create --print-join-command

#加入master节点命令
kubeadm join 10.20.0.200:6443 --token dfkd.skksksk \
    --discovery-token-ca-cert-hash sha256:sssddgfffff4444 \
    --control-plane --certificate-key fffdddddaaa

#加入worker节点
kubeadm join 10.20.0.200:6443 --token dfkd.skksksk \
    --discovery-token-ca-cert-hash sha256:sssddgfffff4444

#初始化其他master加入集群

查看节点状态

kubectl get nodes
kubectl get pods -n kube-system -o wide

初始化其他master节点

kubeadm join 10.20.0.200:6443 --token dfkd.skksksk \
    --discovery-token-ca-cert-hash sha256:sssddgfffff4444 \
    --control-plane --certificate-key fffdddddaaa

加入worker节点

#加入worker节点
kubeadm join 10.20.0.200:6443 --token dfkd.skksksk \
    --discovery-token-ca-cert-hash sha256:sssddgfffff4444

查看集群状态（节点不可用，是因为没有安装网络组件）

[root@k8s-master01]# kubectl  get node

NAME           STATUS     ROLES                  AGE     VERSION
k8s-master01   NotReady   control-plane,master   8m53s   v1.23.5
k8s-master02   NotReady   control-plane,master   2m25s   v1.23.5
k8s-master03   NotReady   control-plane,master   31s     v1.23.5
k8s-node01     NotReady   <none>                 32s     v1.20.0
k8s-node02     NotReady   <none>                 88s    v1.23.5

calico安装

curl https://docs.projectcalico.org/manifests/calico.yaml -o /root/calico.yaml

sed -i 's#docker.io/calico/cni:v3.22.2#registry.cn-shanghai.aliyuncs.com/cni:v3.22.2#' /root/i/calico.yaml
sed -i 's#docker.io/calico/pod2daemon-flexvol:v3.22.2#registry.cn-shanghai.aliyuncs.com/pod2daemon-flexvol:v3.22.2#' /root/calico.yaml
sed -i 's#docker.io/calico/node:v3.22.2#registry.cn-shanghai.aliyuncs.com/node:v3.22.2#' /root/calico.yaml
sed -i 's#docker.io/calico/kube-controllers:v3.22.2#registry.cn-shanghai.aliyuncs.com/kube-controllers:v3.22.2#' /root/calico.yaml

kubectl apply -f /root/calico.yaml

查看集群状态

[root@k8s-master03 ~]# kubectl get node 
NAME          STATUS   ROLES    AGE   VERSION
k8s-master01   Ready    <none>   62d   v1.23.5
k8s-master02   Ready    <none>   62d   v1.23.5
k8s-master03   Ready    <none>   62d   v1.23.5
k8s-node01     Ready    <none>   62d   v1.23.5
k8s-node02     Ready    <none>   62d   v1.23.5

后续安装metric-server、dashboard、ingress-nginx即可