k8s集群部署

星� 星

已于 2022-04-01 10:14:29 修改

阅读量1.7k

点赞数

分类专栏： gaojie 文章标签： linux

于 2022-03-31 22:29:39 首次发布

本文链接：https://blog.csdn.net/weixin_44643790/article/details/123869788

版权

gaojie 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

总思路

前言
使用步骤
`到此集群配置完成`

前言

kubeadm安装工具，提供kubeadm init和kubeadm join，用于快速部署Kubernetes集群。
官方地址：https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm/

使用步骤

1.准备环境

1.关闭防火墙、seliunx；
2.更改主机名，添加本地解析；
3.固定IP，同步时间；
4.关闭swap分区。

systemctl stop firewalld && systemctl disable firewalld
setenforce 0 && sed -i 's/SELINUX=permissive/SELINUX=disabled/' /etc/sysconfig/selinux
hostnamectl set-hostname 
cat >>/etc/hosts <<EOF
192.168.2.13 k8s-master
192.168.2.14 k8s-node1
192.168.2.15 k8s-node2
EOF
swapoff -a && sed -i 's/.*swap.*/#&/' /etc/fstab
#直接用域名同步中国上海时间 是阿里云的服务器
#timedatectl set-timezone Asia/Shanghai 
yum install -y ntpdate
ntpdate ntp1.aliyun.com

2.docker部署

yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-selinux docker-engine-selinux docker-engine

yum install -y yum-utils device-mapper-persistent-data lvm2 git
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum install -y docker-ce
systemctl start docker && systemctl enable docker

阿里镜像下载

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.20.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.20.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.20.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.20.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.13-0
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
docker images

`更换成官方标签，需对应后续版本`

#k8s集群机器通讯
docker pull quay.io/coreos/flannel:v0.14.0

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.20.2 k8s.gcr.io/kube-controller-manager:v1.20.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.20.2 k8s.gcr.io/kube-proxy:v1.20.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.20.2 k8s.gcr.io/kube-apiserver:v1.20.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.20.2 k8s.gcr.io/kube-scheduler:v1.20.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0 k8s.gcr.io/coredns:1.7.0
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.13-0 k8s.gcr.io/etcd:3.4.13-0
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 k8s.gcr.io/pause:3.2

3.kubeadm部署

配置yum源

cat >/etc/yum.repos.d/kubernetes.repo <<EOF 
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubelet-1.20.2-0.x86_64 kubeadm-1.20.2-0.x86_64 kubectl-1.20.2-0.x86_64 ipvsadm
#安装对应版本

安装最新版本
yum makecache fast
yum install -y kubelet kubeadm kubectl ipvsadm

cat >>/etc/rc.local <<EOF
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
EOF
chmod +x /etc/rc.local

cat >/etc/sysctl.d/k8s.conf <<EOF 
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF

reboot重启服务器

---------------------【查看加载成功再继续】-------------------
[root@k8s-master ~]# lsmod | grep ip_vs
ip_vs_sh               12688  0 
ip_vs_wrr              12697  0 
ip_vs_rr               12600  0 
ip_vs                 141092  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          133387  2 ip_vs,nf_conntrack_ipv4
libcrc32c              12644  3 xfs,ip_vs,nf_conntrack
#如果net.bridge.bridge-nf-call-iptables报错，加载br_netfilter模块
#modprobe br_netfilter
#sysctl -p /etc/sysctl.d/k8s.conf

配置变量及配置kubelet的cgroups
DOCKER_CGROUPS=`docker info |grep 'Cgroup' | awk ' NR==1 {print $3}'`

cat >/etc/sysconfig/kubelet<<EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=$DOCKER_CGROUPS --pod-infra-container-image=k8s.gcr.io/pause:3.2"
EOF
启动
systemctl daemon-reload
systemctl enable kubelet && systemctl restart kubelet

在这里使用 # systemctl status kubelet，你会发现报错误信息；
运行 # journalctl -xefu kubelet 命令查看systemd日志：
unable to load client CA file /etc/kubernetes/pki/ca.crt: open /etc/kubernetes/pki/ca.crt: no such file or directory
#这个错误在运行kubeadm init 生成CA证书后会被自动解决，此处可先忽略。
#简单地说就是在kubeadm init 之前kubelet会不断重启。

4.master配置

`记住初始化最后两行输出内容，node执行`

kubeadm init --kubernetes-version=v1.20.2 --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=192.168.2.13  #--ignore-preflight-errors=Swap  加了则忽略swap启动错误
#kubeadm join 192.168.2.13:6443 --token u1wglr.04dkbail6bhlr9tv \
#    --discovery-token-ca-cert-hash sha256:06e65afe18e3f9cab309dd23eccb37e95797a7cc44b96892db8d4d069959a491
rm -rf $HOME/.kube
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes #查看node节点,没准备好

如果报错会有版本提示，那就是有更新新版本了
apiserver-advertise-address=192.168.2.13 —master的ip地址。
–kubernetes-version=v1.20.2 --根据具体版本进行修改
注意在检查一下swap分区是否关闭
初始化输出的内容，根据输出的内容基本上可以看出手动初始化安装一个Kubernetes集群所需要的关键步骤。其中有以下关键内容：
[kubelet] 生成kubelet的配置文件”/var/lib/kubelet/config.yaml”
[certificates]生成相关的各种证书
[kubeconfig]生成相关的kubeconfig文件
[bootstraptoken]生成token记录下来，后边使用kubeadm join往集群中添加节点时会用到

配置网络插件集群互通

1.下载配置

下载flannel文件由于网站被墙了，需要如下操作：
cat >> /etc/hosts <<EOF
199.232.68.133 raw.githubusercontent.com
EOF
curl -O https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
如无法下载，请转本站下载：https://download.csdn.net/download/weixin_44643790/85061403
修改配置文件kube-flannel.yml:
此处的ip配置要与上面kubeadm的pod-network一致，本来就一致，不用改
net-conf.json: |
{
“Network”: “10.244.0.0/16”,
“Backend”: {
“Type”: “vxlan”
}
}

#这里注意kube-flannel.yml这个文件里的flannel的镜像是quay.io/coreos/flannel:v0.14.0 需要提前pull下来。
#如果Node有多个网卡的话，参考https://github.com/kubernetes/kubernetes/issues/39701
#目前需要在kube-flannel.yml中使用–iface参数指定集群主机内网网卡的名称，否则可能会出现dns无法解析。容器无法通信的情况。
#需要将kube-flannel.yml下载到本地，
#flanneld启动参数加上–iface=
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.12.0-amd64
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
- --iface=ens33
- --iface=eth0
⚠️⚠️⚠️–iface=ens33 的值，是你当前的网卡,或者可以指定多网卡

#1.12版本的kubeadm额外给node1节点设置了一个污点(Taint)：node.kubernetes.io/not-ready:NoSchedule，
#很容易理解，即如果节点还没有ready之前，是不接受调度的。可是如果Kubernetes的网络插件还没有部署的话，节点是不会进入ready状态的。
#因此修改以下kube-flannel.yaml的内容，加入对node.kubernetes.io/not-ready:NoSchedule这个污点的容忍：
- key: beta.kubernetes.io/arch
operator: In
values:
- arm64
hostNetwork: true
tolerations:
- operator: Exists
effect: NoSchedule
- key: node.kubernetes.io/not-ready #添加如下三行—在165行左右
operator: Exists
effect: NoSchedule
serviceAccountName: flannel

2.启动

kubectl apply -f flannel.yaml
#启动完成之后需要等待一会,预计100秒全部Running
查看：kubectl get pods --namespace kube-system
NAME READY STATUS RESTARTS AGE
coredns-74ff55c5b-ddjkr 1/1 Running 0 65m
coredns-74ff55c5b-l7klr 1/1 Running 0 65m
etcd-k8s-master 1/1 Running 0 66m
kube-apiserver-k8s-master 1/1 Running 0 66m
kube-controller-manager-k8s-master 1/1 Running 0 66m
kube-flannel-ds-qrd8r 1/1 Running 0 100s
kube-proxy-6s5p8 1/1 Running 0 66m
kube-scheduler-k8s-master 1/1 Running 0 66m
#kubectl get service
#kubectl get svc --namespace kube-system
只有网络插件也安装配置完成之后，才能会显示为ready状态

5.node配置加入集群

加入集群，如果报错开启ip转发：
#sysctl -w net.ipv4.ip_forward=1
在所有node节点操作，此命令为初始化master成功后返回的结果
#kubeadm join 192.168.2.13:6443 --token u1wglr.04dkbail6bhlr9tv
–discovery-token-ca-cert-hash sha256:06e65afe18e3f9cab309dd23eccb37e95797a7cc44b96892db8d4d069959a491

6.master检查

各种检测：
1.查看pods:
[root@k8s-master ~]# kubectl get pods --namespace kube-system
NAME READY STATUS RESTARTS AGE
coredns-74ff55c5b-ddjkr 1/1 Running 0 83m
coredns-74ff55c5b-l7klr 1/1 Running 0 83m
etcd-k8s-master 1/1 Running 0 84m
kube-apiserver-k8s-master 1/1 Running 0 84m
kube-controller-manager-k8s-master 1/1 Running 0 84m
kube-flannel-ds-9sv7h 1/1 Running 0 9m37s
kube-flannel-ds-ctc9d 1/1 Running 0 9m48s
kube-flannel-ds-qrd8r 1/1 Running 0 19m
kube-proxy-44fh7 1/1 Running 0 9m48s
kube-proxy-6s5p8 1/1 Running 0 83m
kube-proxy-jmmfm 1/1 Running 0 9m37s
kube-scheduler-k8s-master 1/1 Running 0 84m
2.查看节点
[root@k8s-master ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane,master 90m v1.20.2
k8s-node1 Ready 16m v1.20.2
k8s-node2 Ready 15m v1.20.2
《本页以下可不用》3.查看异常pod信息：
[root@k8s-master ~]# kubectl describe pods kube-flannel-ds-qrd8r -n kube-system
Name: kube-flannel-ds-qrd8r
Namespace: kube-system
Priority: 2000001000
Priority Class Name: system-node-critical
…
Events:
Type Reason Age From Message ---- ------ ---- ---- -------
Normal Scheduled 22m default-scheduler Successfully assigned kube-system/kube-flannel-ds-qrd8r to k8s-master
Warning Failed 22m kubelet Failed to pull image “rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1”: rpc error: code = Unknown desc = Get “https://registry-1.docker.io/v2/rancher/mirrored-flannelcni-flannel-cni-plugin/manifests/sha256:e2396315d992b15a5e311e4ca76dfd0cfc9f98a91df4b22367f4b2d70ac69370”: dial tcp: lookup registry-1.docker.io on 114.114.114.114:53: read udp 192.168.2.13:38272->114.114.114.114:53: i/o timeout
Warning Failed 22m kubelet Error: ErrImagePull
Normal BackOff 22m kubelet Back-off pulling image “rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1”
Warning Failed 22m kubelet Error: ImagePullBackOff
Normal Pulling 22m (x2 over 22m) kubelet Pulling image “rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1”
Normal Pulled 21m kubelet Successfully pulled image “rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1” in 21.740640691s
Normal Created 21m kubelet Created container install-cni-plugin
Normal Started 21m kubelet Started container install-cni-plugin
Normal Pulled 21m kubelet Container image “quay.io/coreos/flannel:v0.14.0” already present on machine
Normal Created 21m kubelet Created container install-cni
Normal Started 21m kubelet Started container install-cni
Normal Pulled 21m kubelet Container image “quay.io/coreos/flannel:v0.14.0” already present on machine
Normal Created 21m kubelet Created container kube-flannel
Normal Started 21m kubelet Started container kube-flannel
4.遇到这种情况直接删除异常pod:
[root@kub-k8s-master ~]# kubectl delete pod kube-flannel-ds-qrd8r -n kube-system pod “kube-flannel-ds-qrd8r” deleted
再次查看kubectl get pods --namespace kube-system
kube-flannel-ds-fngm4 1/1 Running 0 79s
kube-flannel-ds-fx4rb 1/1 Running 0 30s
kube-flannel-ds-vv6cc 1/1 Running 0 10s

`到此集群配置完成`

-------------以下个人所需-------------

master 添加节点
#重新生成token
kubeadm token create --print-join-command
#重新生成证书
kubeadm init phase upload-certs --upload-certs

新节点加入集群
kubeadm join 192.168.2.14:6443 --token vtxxim.sy0u93t20ixpg4sq --discovery-token-ca-cert-hash sha256:308629a4406bfca94585345d0d15c00d95a9876bf772386cb3d54e9482af6fea
#也可以添加master 节点
#添加新master节点
kubeadm join apiserver.cluster.local:6443 --token sc2ty3.ej38ceisi5lmt9ad --discovery-token-ca-cert-hash sha256:42bf6e526b795854b61b7c0ca875f9a8292b989d44f0f51a4d8dec450711b89e --control-plane --certificate-key 0c00611d30adffe68126477aa33613604c4a423ae2c06e125fe55f838a88b45f

删除node节点
#驱离node节点上的pod
kubectl drain k8s-node3 --delete-local-data --force --ignore-daemonsets
#检查节点状态，被标记为不可调度节点
kubectl get nodes
#删除这个node节点
kubectl delete node k8s-node3>