kubernetes集群部署

最新推荐文章于 2024-06-26 09:38:00 发布

翰泽_henry

最新推荐文章于 2024-06-26 09:38:00 发布

阅读量493

点赞数

文章标签： docker kubernetes 容器

本文链接：https://blog.csdn.net/weixin_36888014/article/details/129700647

版权

1.环境介绍

#网关
#必须开启
Utility: CentOS 7.9     root:root  student:student     192.168.19.254   utility.example.com 功能： gateway,资源，存储，k8s客户端


#k8s集群节点
cka-1:   Ubuntu 20.04    root:root  student:student     192.168.19.101   cka-1.example.com 功能： master(controller node)
cka-2:   Ubuntu 20.04    root:root  student:student     192.168.19.102   cka-2.example.com 功能： node1(worker node)
cka-3:   Ubuntu 20.04    root:root  student:student     192.168.19.103   cka-3.example.com 功能： node2(worker node)

Utility: 工具 ，客户端 ，存储，网关 
cka-1:  master :  kubernetes 集群管理
cka-2:  woker node ： 运行生产pod
cka-3:  woker node ： 运行生产pod

[root@utility ~]# firewall-cmd --list-all --zone=external
external (active)
  target: default
  icmp-block-inversion: no
  interfaces: ens33
  sources:
  services: ssh
  ports:
  protocols:
  masquerade: yes
  forward-ports:
  source-ports:
  icmp-blocks:
  rich rules:

2.安装和配置kubernetes集群

Kubernetes部署方式：

1）kubeadm   (实验和生产环境)  #基于 容器 部署 

2）二进制部署 （生产环境） # 可定制性高  基于 本地服务 部署

在所有节点关闭selinux,swap

root@cka-2:~# sestatus -v  #若无setstatus命令则通过apt install -y policycoreutils安装查看
SELinux status:                 disabled

sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab
临时关闭 swapoff -a
永久关闭 sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab    重启生效
swapon -s查看

2.1安装docker

在所有节点上安装部署docker

#在所有节点上安装docker：docker.io=20.10.7-0ubuntu5~20.04.2
更新软件缓存
student@cka-1:~$ sudo apt update
[sudo] password for student: student
student@cka-1:~$ sudo apt-cache madison docker.io
docker.io | 20.10.21-0ubuntu1~20.04.1 | http://mirrors.aliyun.com/ubuntu focal-updates/universe amd64 Packages
 docker.io | 20.10.7-0ubuntu5~20.04.2 | http://mirrors.aliyun.com/ubuntu focal-security/universe amd64 Packages
 docker.io | 19.03.8-0ubuntu1 | http://mirrors.aliyun.com/ubuntu focal/universe amd64 Packages
 docker.io | 19.03.8-0ubuntu1 | http://mirrors.aliyun.com/ubuntu focal/universe Sources
 docker.io | 20.10.7-0ubuntu5~20.04.2 | http://mirrors.aliyun.com/ubuntu focal-security/universe Sources
 docker.io | 20.10.21-0ubuntu1~20.04.1 | http://mirrors.aliyun.com/ubuntu focal-updates/universe Sources
student@cka-1/2/3:~$ sudo apt install -y docker.io=20.10.7-0ubuntu5~20.04.2
#固定docker版本-不让系统自动升级，避免升级后与k8s版本不兼容
student@cka-1:~$ sudo apt-mark hold docker.io
docker.io set on hold.

#配置docker仓库：阿里镜像加速器
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://g3lk3ccj.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

student@cka-1:~$ sudo docker info | grep Registry -A2
WARNING: No swap limit support
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
--
 Registry Mirrors:
  https://g3lk3ccj.mirror.aliyuncs.com/
 Live Restore Enabled: false
 
 
#在所有节点修改内核参数
tee /etc/sysctl.d/docker-conf <<-'EOF'
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
加载内核参数配置
sysctl -p /etc/sysctl.d/docker-conf

#所有节点配置docker cgroup 支持kubelet
sudo sed -i "s#^ExecStart=/usr/bin/dockerd.*#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd#g" /usr/lib/systemd/system/docker.service
student@cka-2:~$ sudo systemctl daemon-reload
student@cka-2:~$ sudo systemctl restart docker

2.2部署kubernetes

部署kubernetes集群： kubeadm 方式部署

#在所有节点上配置阿里kubernetes仓库：加速
#下载阿里kubernetes软件仓库的公钥，用于验证软件包的签名

student@cka-1:~$ sudo curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -


#在所有节点上添加 阿里 kuberneter源：
#配置阿里kubernetes软件仓库：

sudo tee /etc/apt/sources.list.d/kubernetes.list <<EOF 
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

#更新软件缓存
sudo apt update

--------------------------------
kubernetes软件包：
kubeadm: 部署工具及升级工具
kubelet: kubernetes node agent # k8s节点代理  ------>  管理runtime:  docker  ,containerd, cri 
kubectl : 客户端工具： kubectl  # 只要网络可达， 有kubeconf文件就可以远程管理

#升级  不做大版本升级，只做小版本升级 1.21.1-00----->1.21.8-00
#安装kubeadm,kubelet,kubectl #部署 1.21.0-00
#kubeadm,kubelet,kubectl  三个版本要一致

#查询可以部署版本：
student@cka-3:~$ sudo apt-cache madison kubeadm
   kubeadm |  1.24.0-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.23.6-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.23.5-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.23.4-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.23.3-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.23.2-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages


#在所有节点上安装对应版本的软件包：

student@cka-1:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00
student@cka-2:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00
student@cka-3:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00


#hold版本：将kubernetes 版本 hold 固定住

sudo apt-mark hold kubeadm kubelet kubectl



部署kubernetes集群：单master节点部署

#只在cka-1上运行：
#输出集群默认配置文件


student@cka-1:~$ sudo kubeadm config print init-defaults
[sudo] password for student:
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: node
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: 1.21.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}


初始化master: #在cka-1： 192.168.19.101 初始化

sudo kubeadm init --kubernetes-version=1.21.0 \
--apiserver-advertise-address=192.168.19.101 \
--image-repository registry.aliyuncs.com/google_containers \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.244.0.0/16

参数：

--kubernetes-version=1.21.0 ： 指定安装  1.21.0  最新  1.24.0 

--apiserver-advertise-address=192.168.19.101  # master  

--image-repository registry.aliyuncs.com/google_containers  # 安装时下载 阿里的镜像

--service-cidr=10.1.0.0/16  #  svc的 网络  VIP   # kubeproxy  #ClusterIP

--pod-network-cidr=10.244.0.0/16  #  pod网段

#报错处理
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
error execution phase preflight: [preflight] Some fatal errors occurred:
        [ERROR ImagePull]: failed to pull image registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0: output: Error response from daemon: pull access denied for registry.aliyuncs.com/google_containers/coredns/coredns, repository does not exist or may require 'docker login': denied: requested access to the resource is denied
, error: exit status 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher
#解决方法
student@cka-1:~$ sudo docker pull registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0
Error response from daemon: pull access denied for registry.aliyuncs.com/google_containers/coredns/coredns, repository does not exist or may require 'docker login': denied: requested access to the resource is denied
发现在阿里仓库找不到coredns镜像
student@cka-1:~$ sudo docker pull ninokop/coredns:v1.8.0 #这一步三个node都需要做，因为不确定coredns运行在哪个node

student@cka-1:~$ sudo docker images
REPOSITORY                                                        TAG        IMAGE ID       CREATED         SIZE
nginx                                                             latest     605c77e624dd   14 months ago   141MB
registry.aliyuncs.com/google_containers/kube-apiserver            v1.21.0    4d217480042e   23 months ago   126MB
registry.aliyuncs.com/google_containers/kube-proxy                v1.21.0    38ddd85fe90e   23 months ago   122MB
registry.aliyuncs.com/google_containers/kube-controller-manager   v1.21.0    09708983cc37   23 months ago   120MB
registry.aliyuncs.com/google_containers/kube-scheduler            v1.21.0    62ad3129eca8   23 months ago   50.6MB
registry.aliyuncs.com/google_containers/pause                     3.4.1      0f8457a4c2ec   2 years ago     683kB
ninokop/coredns                                                   v1.8.0     296a6d5035e2   2 years ago     42.5MB
registry.aliyuncs.com/google_containers/etcd                      3.4.13-0   0369cf4303ff   2 years ago     253MB

#修改image的tag
student@cka-1:~$ sudo docker tag 296a6d5035e2 registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0
student@cka-1:~$ sudo docker rmi ninokop/coredns:v1.8.0
#补充，可将coredns镜像保存到本地，后续使用可直接加载
student@cka-1:~$ sudo docker images | grep coredns
registry.aliyuncs.com/google_containers/coredns/coredns           v1.8.0     296a6d5035e2   2 years ago     42.5MB
student@cka-1:~$ sudo docker save registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0 -o ./coredns.tar
加载镜像命令
docker load -i coredns.tar
#01节点再次执行初始化
sudo kubeadm init --kubernetes-version=1.21.0 \
--apiserver-advertise-address=192.168.19.101 \
--image-repository registry.aliyuncs.com/google_containers \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.244.0.0/16

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

master节点安装完成后发现以下两个问题
student@cka-1:~$ kubectl get nodes
NAME                STATUS     ROLES                  AGE   VERSION
cka-1.example.com   NotReady   control-plane,master   20m   v1.21.0
student@cka-1:~$ kubectl get pods -A | grep -i pending
kube-system   coredns-545d6fc579-pvc2w                    0/1     Pending   0          5h11m
kube-system   coredns-545d6fc579-q68xt                    0/1     Pending   0          5h11m
以上两个问题是因为没有安装cni网络插件
通过以下地址点calico访问calico官网
https://kubernetes.io/docs/concepts/cluster-administration/addons/   -------->
https://docs.tigera.io/calico/latest/getting-started/kubernetes/self-managed-onprem/onpremises
#打印出join worker node token:

kubeadm join 192.168.19.101:6443 --token 7p9neb.02m2aktfzl8903mo \
        --discovery-token-ca-cert-hash sha256:f89ad2175d2f2820efa7f31971ac11e3ac8b6e6b20c289e5e155c7cd4c6a524b
这个命令token的时效性只有24小时，可以通过另一条命令生成
kubeadm token create --print-join-command
#cka-2/3加入集群
student@cka-2:~$ sudo kubeadm join 192.168.19.101:6443 --token px2c8o.iuacqclk1xrcfget --discovery-token-ca-cert-hash sha256:f89ad2175d2f2820efa7f31971ac11e3ac8b6e6b20c289e5e155c7cd4c6a524b

student@cka-1:~$ kubectl get nodes
NAME                STATUS     ROLES                  AGE     VERSION
cka-1.example.com   NotReady   control-plane,master   5h38m   v1.21.0
cka-2.example.com   NotReady   <none>                 66s     v1.21.0
cka-3.example.com   NotReady   <none>                 7s      v1.21.0

cni: container network interface

部署calico 网络组件 支持 networkpolicy  

#下载部署calico yaml文件
student@cka-1:~$ wget https://docs.projectcalico.org/v3.20/manifests/calico.yaml
--2023-03-21 12:39:32--  https://docs.projectcalico.org/v3.20/manifests/calico.yaml
Resolving docs.projectcalico.org (docs.projectcalico.org)... 52.74.166.77, 34.126.184.144, 2406:da18:880:3800::c8, ...
Connecting to docs.projectcalico.org (docs.projectcalico.org)|52.74.166.77|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 202824 (198K) [text/yaml]
Saving to: ‘calico.yaml’

calico.yaml                                        100%[===============================================================================================================>] 198.07K   284KB/s    in 0.7s

2023-03-21 12:39:33 (284 KB/s) - ‘calico.yaml’ saved [202824/202824]

#查看yaml编排中的image有哪些？
student@cka-1:~$ ls
calico.yaml  coredns.tar  LFS258  LFS258.tar

student@cka-1:~$ cat calico.yaml |grep image
          image: docker.io/calico/cni:v3.20.6
          image: docker.io/calico/cni:v3.20.6
          image: docker.io/calico/pod2daemon-flexvol:v3.20.6
          image: docker.io/calico/node:v3.20.6
          image: docker.io/calico/kube-controllers:v3.20.6
student@cka-1:~$ 


#在cka-1,cka-2,cka-3上下载calico docker image   
sudo docker pull calico/cni:v3.20.6
sudo docker pull calico/pod2daemon-flexvol:v3.20.6
sudo docker pull calico/node:v3.20.6
sudo docker pull calico/kube-controllers:v3.20.6
student@cka-1:~$ sudo docker images
REPOSITORY                                                        TAG        IMAGE ID       CREATED         SIZE
calico/node                                                       v3.20.6    daeec7e26e1f   7 months ago    156MB
calico/pod2daemon-flexvol                                         v3.20.6    39b166f3f936   7 months ago    18.6MB
calico/cni                                                        v3.20.6    13b6f63a50d6   7 months ago    138MB
calico/kube-controllers                                           v3.20.6    4dc6e7685020   7 months ago    60.2MB
nginx                                                             latest     605c77e624dd   14 months ago   141MB
registry.aliyuncs.com/google_containers/kube-apiserver            v1.21.0    4d217480042e   23 months ago   126MB
registry.aliyuncs.com/google_containers/kube-proxy                v1.21.0    38ddd85fe90e   23 months ago   122MB
registry.aliyuncs.com/google_containers/kube-scheduler            v1.21.0    62ad3129eca8   23 months ago   50.6MB
registry.aliyuncs.com/google_containers/kube-controller-manager   v1.21.0    09708983cc37   23 months ago   120MB
registry.aliyuncs.com/google_containers/pause                     3.4.1      0f8457a4c2ec   2 years ago     683kB
registry.aliyuncs.com/google_containers/coredns/coredns           v1.8.0     296a6d5035e2   2 years ago     42.5MB
registry.aliyuncs.com/google_containers/etcd                      3.4.13-0   0369cf4303ff   2 years ago     253MB
#部署calico
student@cka-1:~$ kubectl create -f ./calico.yaml

查看node/pod状态

student@cka-1:~$ kubectl get nodes
NAME                STATUS   ROLES                  AGE    VERSION
cka-1.example.com   Ready    control-plane,master   8h     v1.21.0
cka-2.example.com   Ready    <none>                 173m   v1.21.0
cka-3.example.com   Ready    <none>                 172m   v1.21.0
student@cka-1:~$ kubectl get pods -n kube-system
NAME                                        READY   STATUS    RESTARTS   AGE
calico-kube-controllers-594649bd75-dgs5z    1/1     Running   0          90s
calico-node-72vk7                           1/1     Running   0          90s
calico-node-84tbc                           1/1     Running   0          90s
calico-node-qbh8q                           1/1     Running   0          90s
coredns-545d6fc579-pvc2w                    1/1     Running   0          8h
coredns-545d6fc579-q68xt                    1/1     Running   0          8h
etcd-cka-1.example.com                      1/1     Running   1          8h
kube-apiserver-cka-1.example.com            1/1     Running   1          8h
kube-controller-manager-cka-1.example.com   1/1     Running   1          8h
kube-proxy-c96sv                            1/1     Running   1          174m
kube-proxy-nv4mv                            1/1     Running   1          173m
kube-proxy-tpb6b                            1/1     Running   1          8h
kube-scheduler-cka-1.example.com            1/1     Running   1          8h

2.3补充

#配置kubectl命令自动补全
student@cka-1:~$ sudo apt install -y bash-completion
student@cka-1:~$ source <(kubectl completion bash) #临时有效 ，重启无效
#将source <(kubectl completion bash) 当前用户的~/.bashrc
student@cka-1:~$ echo "source <(kubectl completion bash)" >> ~/.bashrc 
重启有效

配置客户端
#配置utility为kubernetes客户端：

配置centos kubernetes的仓库：
[student@utility ~]$ sudo -i
[root@utility ~]# cat <<EOF > /etc/yum.repos.d/kubernetes.repo
> [kubernetes]
> name=Kubernetes
> baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
> enabled=1
> gpgcheck=0
> EOF
[root@utility ~]# yum clean all
[root@utility ~]# yum makecache
#查找仓库中的kubectl 软件列表
[root@utility ~]# yum list --showduplicates kubectl --disableexcludes=kubernetes | grep '1.21.0'
kubectl.x86_64                       1.21.0-0                         kubernetes

#安装指定软件版本
[root@utility ~]# yum -y install kubectl-1.21.0-0

#kubectl连接集群
#创建 密钥目录：~/.kube
[student@utility ~]$ mkdir ~/.kube
[student@utility ~]$ scp student@cka-1:~/.kube/config ./.kube/
[student@utility ~]$ chmod 600 .kube/config
[student@utility ~]$ source <(kubectl completion bash)
[student@utility ~]$ echo "source <(kubectl completion bash)" >> ~/.bashrc
[student@utility ~]$ kubectl get nodes
NAME                STATUS   ROLES                  AGE     VERSION
cka-1.example.com   Ready    control-plane,master   9h      v1.21.0
cka-2.example.com   Ready    <none>                 3h36m   v1.21.0
cka-3.example.com   Ready    <none>                 3h35m   v1.21.0