Centos7.9上用kubeadm搭建1.23.6版本Kubernetes集群

#Centos7.9上用kubeadm搭建1.23.6版本Kubernetes集群

一、部署节点说明

系统	主机名	外部IP地址	内部IP地址
Centos 7.9.2009	master01	192.168.3.31	172.18.3.31
Centos 7.9.2009	node01	192.168.3.41	172.18.3.41
Centos 7.9.2009	node02	192.168.3.42	172.18.3.42
Centos 7.9.2009	node03	192.168.3.43	172.18.3.43

二、使用下面脚本初始化配置各个节点

#!/bin/sh
KUBVERSION=1.23.6

#安装工具软件
yum install vim net-tools wget lsof ipset ipvsadm -y

#关闭防火墙和selinux
systemctl stop firewalld && systemctl disable firewalld
sed -i '/^SELINUX=/c SELINUX=disabled' /etc/selinux/config
setenforce 0

#关闭swap
swapoff -a
sed -i 's/^.*centos-swap/#&/g' /etc/fstab

#配置主机地址解析
if [ -z "`cat /etc/hosts | grep \`ip route ls | grep 192.168.3.0/24 | awk '{print $9}'\` | awk '{print $2}'`" ]; then
cat << EOF >> /etc/hosts
192.168.3.30 master
192.168.3.31 master01
192.168.3.32 master03
192.168.3.33 master03
192.168.3.41 node01
192.168.3.42 node02
192.168.3.43 node03
EOF
fi

#根据IP地址获取主机名并写入hostname
echo `cat /etc/hosts | grep \`ip route ls | grep 192.168.3.0/24 | awk '{print $9}'\` | awk '{print $2}'` >/etc/hostname

#重新登录终端立即生效
hostnamectl set-hostname `cat /etc/hosts | grep \`ip route ls | grep 192.168.3.0/24 | awk '{print $9}'\` | awk '{print $2}'`

# 激活 br_netfilter 模块
modprobe br_netfilter
cat << EOF > /etc/modules-load.d/k8s.conf
br_netfilter
EOF

# 内核参数设置：开启IP转发，允许iptables对bridge的数据进行处理
cat << EOF > /etc/sysctl.d/k8s.conf 
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 立即生效
sysctl --system

#开启 ipvs,不开启 ipvs 将会使用 iptables，但是效率低，所以官网推荐需要开通 ipvs 内核

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4


#配置集群时间同步
yum install -y chrony

#master节点：
if [ ! -z "`cat /etc/hosts | grep \`ip route ls | grep 192.168.3.0/24 | awk '{print $9}'\` | grep master `" ]; then
cat > /etc/chrony.conf << EOF
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
logdir /var/log/chrony
server ntp1.aliyun.com iburst
local stratum 10
allow 192.168.3.0/24
EOF
systemctl restart chronyd
systemctl enable chronyd
fi

#node节点
if [ ! -z "`cat /etc/hosts | grep \`ip route ls | grep 192.168.3.0/24 | awk '{print $9}'\` | grep node `" ]; then
cat > /etc/chrony.conf << EOF
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
logdir /var/log/chrony
server 192.168.3.31 iburst
server 192.168.3.32 iburst
server 192.168.3.33 iburst
EOF
systemctl restart chronyd
systemctl enable chronyd
fi


#查看同步状态：
chronyc sources -v

#安装部署docker
#1.安装docker-ce
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y docker-ce
systemctl enable docker && systemctl start docker

#修改docker-ce默认配置为systemd
cat << EOF > /etc/docker/daemon.json
{
  "storage-driver": "overlay2",
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl daemon-reload
systemctl restart docker


#安装 kubeadm 和相关工具
#由于官方源位于国外，这里配置centos7 kubernetes国内阿里源
cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#安装 kubeadm 工具
yum install -y kubelet-$KUBVERSION kubeadm-$KUBVERSION kubectl-$KUBVERSION
cat <<EOF >/etc/sysconfig/kubelet
KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
EOF
systemctl enable kubelet && systemctl start kubelet

三、执行下面脚本升级ELRepo内核，不升级内核不支持ipvlan，部分网络会有问题

#!/bin/sh
#载入公钥
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

#查看可用的kernel包
yum --disablerepo=\* --enablerepo=elrepo-kernel list kernel*

#安装最新lt版本的kernel
yum --disablerepo=\* --enablerepo=elrepo-kernel install  kernel-lt.x86_64  -y

#删除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64  -y

#安装新版本工具包
yum --disablerepo=\* --enablerepo=elrepo-kernel install kernel-lt-tools.x86_64  -y

#查看内核插入顺序
awk -F \' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg

#设置默认启动
grub2-set-default `awk -F \' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
| grep elrepo | head -n1 | awk '{print $1}'`

#查看当前实际启动顺序
grub2-editenv list

#重启并检查内核升级情况,<br>
root@master01 ~]# uname -r
5.4.195-1.el7.elrepo.x86_64
[root@master01 ~]# 

四、安装 master 节点
1.初始化 kubernetns master01 节点

kubeadm init  \
  --image-repository=registry.aliyuncs.com/google_containers  \
  --kubernetes-version 1.23.6                                 \
  --apiserver-advertise-address=192.168.3.31                  \
  --service-cidr=172.18.0.0/16                                \
  --pod-network-cidr=10.244.0.0/16                            \
  --v=5

选项说明：
  --image-repository：选择用于拉取镜像的镜像仓库（默认为“k8s.gcr.io” ）
  --kubernetes-version：选择特定的Kubernetes版本（默认为“stable-1”）
  --service-cidr：为服务的VIP指定使用的IP地址范围（默认为“10.96.0.0/12”）
  --pod-network-cidr：指定Pod网络的IP地址范围。如果设置，则将自动为每个节点分配CIDR。

注：
  因为后面要部署 flannel，参照flannel文档，我们要指定Pod网络的IP地址范围为10.244.0.0/16

2.输出内容，可以看到初始化成功的信息和一些提示

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.3.31:6443 --token 2z0fhd.utyva9i2p3vu5ckd \
        --discovery-token-ca-cert-hash sha256:c6f3981005cb5d6e08e908bb278151dddddb18c98b06f0fcf51d0220e44c6884 

3.根据上面提示内容执行如下操作

# 要开始使用集群，您需要以常规用户身份运行以下命令
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 或者，如果您是root用户，则可以运行允许命令
export KUBECONFIG=/etc/kubernetes/admin.conf

# 加入.bashrc，方便以后连接服务器自动执行
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >>/root/.bashrc

四、安装网络插件，有问题请科学上网

kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml

警告信息，可以忽略

[root@master01 ~]# kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
[root@master01 ~]# 

# 查看flannel状态
[root@master01 ~]# kubectl get pods -A | grep flannel
kube-system   kube-flannel-ds-d6zfq              1/1     Running   0          28s
[root@master01 ~]# 

kubectl describe pod kube-flannel-ds-d6zfq -n kube-system
#直到状态变为Running

五、添加 3个Node节点到集群

# 在master01上获取添加方式
[root@master01 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.3.31:6443 --token rnszep.ivrl22zv30ah3r0n --discovery-token-ca-cert-hash sha256:c6f3981005cb5d6e08e908bb278151dddddb18c98b06f0fcf51d0220e44c6884 
[root@master01 ~]# 

# 在node01、node02、node03上分别执行添加命令

#在master01上验证
kubectl get nodes
kubectl get nodes -o wide
kubectl get pods --all-namespaces

#直到全部变为Ready
[root@master01 ~]# kubectl get nodes
NAME       STATUS   ROLES                  AGE     VERSION
master01   Ready    control-plane,master   6m27s   v1.23.6
node01     Ready    <none>                 88s     v1.23.6
node02     Ready    <none>                 84s     v1.23.6
node03     Ready    <none>                 78s     v1.23.6
[root@master01 ~]# 

六、部署dashboard
  Dashboard 是基于网页的 Kubernetes 用户界面。您可以使用 Dashboard 将容器应用部署到 Kubernetes 集群中，也可以对容器应用排错，还能管理集群本身及其附属资源。您可以使用 Dashboard 获取运行在集群中的应用的概览信息，也可以创建或者修改 Kubernetes 资源（如 Deployment，Job，DaemonSet 等等）。例如，您可以对 Deployment 实现弹性伸缩、发起滚动升级、重启 Pod 或者使用向导创建新的应用。
   安装dashboard：（https://github.com/kubernetes/dashboard）

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.1/aio/deploy/recommended.yaml

#耐心等待状态变为Running
kubectl get pods -n kubernetes-dashboard


# 修改对外暴露端口
[root@master01 ~]# kubectl edit svc -n kubernetes-dashboard kubernetes-dashboard
将 type: ClusterIP 修改为 type: NodePort 即可


# 获取对外暴露端口
[root@master01 ~]# kubectl get svc -n kubernetes-dashboard
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)         AGE
dashboard-metrics-scraper   ClusterIP   172.18.149.94    <none>        8000/TCP        111s
kubernetes-dashboard        NodePort    172.18.130.174   <none>        443:31230/TCP   111s
[root@master01 ~]# 

使用浏览器访问：
https://192.168.3.31:31230/#/login

创建服务用户，集群角色绑定，然后获取token
[root@master01 ~]# cat << EOF >token.yaml 
apiVersion : v1
kind : ServiceAccount
metadata :
  name : admin-user
  namespace : kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF


[root@master01 ~]# kubectl apply -f token.yaml
serviceaccount/admin-user created
clusterrolebinding.rbac.authorization.k8s.io/admin-user created
[root@master01 ~]# 

#获取令牌
kubectl -n kubernetes-dashboard get secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}"

[root@master01 ~]# kubectl -n kubernetes-dashboard get secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}"
eyJhbGciOiJSUzI1NiIsImtpZCI6IkFLQkR6VnU5NHdOakpBNU5kcVdZd3pRZmlhUFNueV8yV19JVmVfaGlxeDgifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWh2NzR2Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJiMzg2NGQ3OC1iZDdmLTQ4N2UtOWY2OS04OWJhZjUxMzNmZjEiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.XJz3R7SV_bi1zr83Yl0c6RnboLzxsTVYmDzJJrJWoKBf8o0AAvQspUV2hngKumk5x_NI-GT3HnCZNdO1Inah6t92O8YBj4DCzx0ELeRr2tY4dcGjntHREOnvsCFnyDeqrzj0MZmtjdxZCPyAUAgogpnHtH5ljPiUUM48b6kADsFok0RzribJqW1Ta6zmCyZ3hBE4cgI2bD5nrRGslkn4DwQWrNFw2O4AiwR2iLm6CpRNIigBUy819khk9x87mMtPVKv5zbxfojD7eqXrnZL9LihuzZRPsWrZYWpnjIIrzsvDWHT2yMYn_1J7t1z8bG-G4p99LuIm2tk_xi7iYcomdQ[root@master01 ~]

#导入令牌，登录kubernetes-dashboard

七、整个机器配置完毕（单master）

部署nginx服务验证集群
[root@master01 ~]# kubectl create deployment nginx --image=nginx
deployment.apps/nginx created
[root@master01 ~]#

[root@master01 ~]# kubectl expose deployment nginx --port=80 --type=NodePort
service/nginx exposed
[root@master01 ~]# 

[root@master01 ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.1.0.1      <none>        443/TCP        33m
nginx        NodePort    10.1.171.38   <none>        80:30931/TCP   39s
[root@master01 ~]# 


[root@master01 ~]#  curl http://192.168.3.31:31603
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
......
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@master01 ~]# 

kubernetes ClusterIP 10.1.0.1 443/TCP 33m
nginx NodePort 10.1.171.38 80:30931/TCP 39s
[root@master01 ~]#

[root@master01 ~]# curl http://192.168.3.31:31603

......

Thank you for using nginx.

[root@master01 ~]#