1.环境介绍
#网关
#必须开启
Utility: CentOS 7.9 root:root student:student 192.168.19.254 utility.example.com 功能: gateway,资源,存储,k8s客户端
#k8s集群节点
cka-1: Ubuntu 20.04 root:root student:student 192.168.19.101 cka-1.example.com 功能: master(controller node)
cka-2: Ubuntu 20.04 root:root student:student 192.168.19.102 cka-2.example.com 功能: node1(worker node)
cka-3: Ubuntu 20.04 root:root student:student 192.168.19.103 cka-3.example.com 功能: node2(worker node)
Utility: 工具 ,客户端 ,存储,网关
cka-1: master : kubernetes 集群管理
cka-2: woker node : 运行生产pod
cka-3: woker node : 运行生产pod
[root@utility ~]# firewall-cmd --list-all --zone=external
external (active)
target: default
icmp-block-inversion: no
interfaces: ens33
sources:
services: ssh
ports:
protocols:
masquerade: yes
forward-ports:
source-ports:
icmp-blocks:
rich rules:
2.安装和配置kubernetes集群
Kubernetes部署方式: 1)kubeadm (实验和生产环境) #基于 容器 部署 2)二进制部署 (生产环境) # 可定制性高 基于 本地服务 部署
在所有节点关闭selinux,swap
root@cka-2:~# sestatus -v #若无setstatus命令则通过apt install -y policycoreutils安装查看
SELinux status: disabled
sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab
临时关闭 swapoff -a
永久关闭 sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab 重启生效
swapon -s查看
2.1安装docker
在所有节点上安装部署docker
#在所有节点上安装docker:docker.io=20.10.7-0ubuntu5~20.04.2
更新软件缓存
student@cka-1:~$ sudo apt update
[sudo] password for student: student
student@cka-1:~$ sudo apt-cache madison docker.io
docker.io | 20.10.21-0ubuntu1~20.04.1 | http://mirrors.aliyun.com/ubuntu focal-updates/universe amd64 Packages
docker.io | 20.10.7-0ubuntu5~20.04.2 | http://mirrors.aliyun.com/ubuntu focal-security/universe amd64 Packages
docker.io | 19.03.8-0ubuntu1 | http://mirrors.aliyun.com/ubuntu focal/universe amd64 Packages
docker.io | 19.03.8-0ubuntu1 | http://mirrors.aliyun.com/ubuntu focal/universe Sources
docker.io | 20.10.7-0ubuntu5~20.04.2 | http://mirrors.aliyun.com/ubuntu focal-security/universe Sources
docker.io | 20.10.21-0ubuntu1~20.04.1 | http://mirrors.aliyun.com/ubuntu focal-updates/universe Sources
student@cka-1/2/3:~$ sudo apt install -y docker.io=20.10.7-0ubuntu5~20.04.2
#固定docker版本-不让系统自动升级,避免升级后与k8s版本不兼容
student@cka-1:~$ sudo apt-mark hold docker.io
docker.io set on hold.
#配置docker仓库:阿里镜像加速器
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://g3lk3ccj.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
student@cka-1:~$ sudo docker info | grep Registry -A2
WARNING: No swap limit support
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
--
Registry Mirrors:
https://g3lk3ccj.mirror.aliyuncs.com/
Live Restore Enabled: false
#在所有节点修改内核参数
tee /etc/sysctl.d/docker-conf <<-'EOF'
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
加载内核参数配置
sysctl -p /etc/sysctl.d/docker-conf
#所有节点配置docker cgroup 支持kubelet
sudo sed -i "s#^ExecStart=/usr/bin/dockerd.*#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd#g" /usr/lib/systemd/system/docker.service
student@cka-2:~$ sudo systemctl daemon-reload
student@cka-2:~$ sudo systemctl restart docker
2.2部署kubernetes
部署kubernetes集群: kubeadm 方式部署
#在所有节点上配置阿里kubernetes仓库:加速
#下载阿里kubernetes软件仓库的公钥,用于验证软件包的签名
student@cka-1:~$ sudo curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
#在所有节点上添加 阿里 kuberneter源: #配置阿里kubernetes软件仓库:
sudo tee /etc/apt/sources.list.d/kubernetes.list <<EOF
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
#更新软件缓存 sudo apt update -------------------------------- kubernetes软件包: kubeadm: 部署工具及升级工具 kubelet: kubernetes node agent # k8s节点代理 ------> 管理runtime: docker ,containerd, cri kubectl : 客户端工具: kubectl # 只要网络可达, 有kubeconf文件就可以远程管理 #升级 不做大版本升级,只做小版本升级 1.21.1-00----->1.21.8-00 #安装kubeadm,kubelet,kubectl #部署 1.21.0-00 #kubeadm,kubelet,kubectl 三个版本要一致
#查询可以部署版本:
student@cka-3:~$ sudo apt-cache madison kubeadm
kubeadm | 1.24.0-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
kubeadm | 1.23.6-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
kubeadm | 1.23.5-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
kubeadm | 1.23.4-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
kubeadm | 1.23.3-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
kubeadm | 1.23.2-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
#在所有节点上安装对应版本的软件包:
student@cka-1:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00
student@cka-2:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00
student@cka-3:~$ sudo apt install -y kubeadm=1.21.0-00 kubelet=1.21.0-00 kubectl=1.21.0-00
#hold版本:将kubernetes 版本 hold 固定住
sudo apt-mark hold kubeadm kubelet kubectl
部署kubernetes集群:单master节点部署 #只在cka-1上运行: #输出集群默认配置文件
student@cka-1:~$ sudo kubeadm config print init-defaults
[sudo] password for student:
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: node
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: 1.21.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
初始化master: #在cka-1: 192.168.19.101 初始化 sudo kubeadm init --kubernetes-version=1.21.0 \ --apiserver-advertise-address=192.168.19.101 \ --image-repository registry.aliyuncs.com/google_containers \ --service-cidr=10.1.0.0/16 \ --pod-network-cidr=10.244.0.0/16 参数: --kubernetes-version=1.21.0 : 指定安装 1.21.0 最新 1.24.0 --apiserver-advertise-address=192.168.19.101 # master --image-repository registry.aliyuncs.com/google_containers # 安装时下载 阿里的镜像 --service-cidr=10.1.0.0/16 # svc的 网络 VIP # kubeproxy #ClusterIP --pod-network-cidr=10.244.0.0/16 # pod网段 #报错处理 [preflight] You can also perform this action in beforehand using 'kubeadm config images pull' error execution phase preflight: [preflight] Some fatal errors occurred: [ERROR ImagePull]: failed to pull image registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0: output: Error response from daemon: pull access denied for registry.aliyuncs.com/google_containers/coredns/coredns, repository does not exist or may require 'docker login': denied: requested access to the resource is denied , error: exit status 1 [preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...` To see the stack trace of this error execute with --v=5 or higher #解决方法 student@cka-1:~$ sudo docker pull registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0 Error response from daemon: pull access denied for registry.aliyuncs.com/google_containers/coredns/coredns, repository does not exist or may require 'docker login': denied: requested access to the resource is denied 发现在阿里仓库找不到coredns镜像 student@cka-1:~$ sudo docker pull ninokop/coredns:v1.8.0 #这一步三个node都需要做,因为不确定coredns运行在哪个node
student@cka-1:~$ sudo docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest 605c77e624dd 14 months ago 141MB
registry.aliyuncs.com/google_containers/kube-apiserver v1.21.0 4d217480042e 23 months ago 126MB
registry.aliyuncs.com/google_containers/kube-proxy v1.21.0 38ddd85fe90e 23 months ago 122MB
registry.aliyuncs.com/google_containers/kube-controller-manager v1.21.0 09708983cc37 23 months ago 120MB
registry.aliyuncs.com/google_containers/kube-scheduler v1.21.0 62ad3129eca8 23 months ago 50.6MB
registry.aliyuncs.com/google_containers/pause 3.4.1 0f8457a4c2ec 2 years ago 683kB
ninokop/coredns v1.8.0 296a6d5035e2 2 years ago 42.5MB
registry.aliyuncs.com/google_containers/etcd 3.4.13-0 0369cf4303ff 2 years ago 253MB
#修改image的tag student@cka-1:~$ sudo docker tag 296a6d5035e2 registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0 student@cka-1:~$ sudo docker rmi ninokop/coredns:v1.8.0 #补充,可将coredns镜像保存到本地,后续使用可直接加载 student@cka-1:~$ sudo docker images | grep coredns registry.aliyuncs.com/google_containers/coredns/coredns v1.8.0 296a6d5035e2 2 years ago 42.5MB student@cka-1:~$ sudo docker save registry.aliyuncs.com/google_containers/coredns/coredns:v1.8.0 -o ./coredns.tar 加载镜像命令 docker load -i coredns.tar #01节点再次执行初始化 sudo kubeadm init --kubernetes-version=1.21.0 \ --apiserver-advertise-address=192.168.19.101 \ --image-repository registry.aliyuncs.com/google_containers \ --service-cidr=10.1.0.0/16 \ --pod-network-cidr=10.244.0.0/16 mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config master节点安装完成后发现以下两个问题 student@cka-1:~$ kubectl get nodes NAME STATUS ROLES AGE VERSION cka-1.example.com NotReady control-plane,master 20m v1.21.0 student@cka-1:~$ kubectl get pods -A | grep -i pending kube-system coredns-545d6fc579-pvc2w 0/1 Pending 0 5h11m kube-system coredns-545d6fc579-q68xt 0/1 Pending 0 5h11m 以上两个问题是因为没有安装cni网络插件 通过以下地址点calico访问calico官网 https://kubernetes.io/docs/concepts/cluster-administration/addons/ --------> https://docs.tigera.io/calico/latest/getting-started/kubernetes/self-managed-onprem/onpremises #打印出join worker node token: kubeadm join 192.168.19.101:6443 --token 7p9neb.02m2aktfzl8903mo \ --discovery-token-ca-cert-hash sha256:f89ad2175d2f2820efa7f31971ac11e3ac8b6e6b20c289e5e155c7cd4c6a524b 这个命令token的时效性只有24小时,可以通过另一条命令生成 kubeadm token create --print-join-command #cka-2/3加入集群 student@cka-2:~$ sudo kubeadm join 192.168.19.101:6443 --token px2c8o.iuacqclk1xrcfget --discovery-token-ca-cert-hash sha256:f89ad2175d2f2820efa7f31971ac11e3ac8b6e6b20c289e5e155c7cd4c6a524b student@cka-1:~$ kubectl get nodes NAME STATUS ROLES AGE VERSION cka-1.example.com NotReady control-plane,master 5h38m v1.21.0 cka-2.example.com NotReady <none> 66s v1.21.0 cka-3.example.com NotReady <none> 7s v1.21.0
cni: container network interface
部署calico 网络组件 支持 networkpolicy #下载部署calico yaml文件 student@cka-1:~$ wget https://docs.projectcalico.org/v3.20/manifests/calico.yaml --2023-03-21 12:39:32-- https://docs.projectcalico.org/v3.20/manifests/calico.yaml Resolving docs.projectcalico.org (docs.projectcalico.org)... 52.74.166.77, 34.126.184.144, 2406:da18:880:3800::c8, ... Connecting to docs.projectcalico.org (docs.projectcalico.org)|52.74.166.77|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 202824 (198K) [text/yaml] Saving to: ‘calico.yaml’ calico.yaml 100%[===============================================================================================================>] 198.07K 284KB/s in 0.7s 2023-03-21 12:39:33 (284 KB/s) - ‘calico.yaml’ saved [202824/202824] #查看yaml编排中的image有哪些? student@cka-1:~$ ls calico.yaml coredns.tar LFS258 LFS258.tar student@cka-1:~$ cat calico.yaml |grep image image: docker.io/calico/cni:v3.20.6 image: docker.io/calico/cni:v3.20.6 image: docker.io/calico/pod2daemon-flexvol:v3.20.6 image: docker.io/calico/node:v3.20.6 image: docker.io/calico/kube-controllers:v3.20.6 student@cka-1:~$ #在cka-1,cka-2,cka-3上下载calico docker image sudo docker pull calico/cni:v3.20.6 sudo docker pull calico/pod2daemon-flexvol:v3.20.6 sudo docker pull calico/node:v3.20.6 sudo docker pull calico/kube-controllers:v3.20.6 student@cka-1:~$ sudo docker images REPOSITORY TAG IMAGE ID CREATED SIZE calico/node v3.20.6 daeec7e26e1f 7 months ago 156MB calico/pod2daemon-flexvol v3.20.6 39b166f3f936 7 months ago 18.6MB calico/cni v3.20.6 13b6f63a50d6 7 months ago 138MB calico/kube-controllers v3.20.6 4dc6e7685020 7 months ago 60.2MB nginx latest 605c77e624dd 14 months ago 141MB registry.aliyuncs.com/google_containers/kube-apiserver v1.21.0 4d217480042e 23 months ago 126MB registry.aliyuncs.com/google_containers/kube-proxy v1.21.0 38ddd85fe90e 23 months ago 122MB registry.aliyuncs.com/google_containers/kube-scheduler v1.21.0 62ad3129eca8 23 months ago 50.6MB registry.aliyuncs.com/google_containers/kube-controller-manager v1.21.0 09708983cc37 23 months ago 120MB registry.aliyuncs.com/google_containers/pause 3.4.1 0f8457a4c2ec 2 years ago 683kB registry.aliyuncs.com/google_containers/coredns/coredns v1.8.0 296a6d5035e2 2 years ago 42.5MB registry.aliyuncs.com/google_containers/etcd 3.4.13-0 0369cf4303ff 2 years ago 253MB #部署calico student@cka-1:~$ kubectl create -f ./calico.yaml
查看node/pod状态
student@cka-1:~$ kubectl get nodes NAME STATUS ROLES AGE VERSION cka-1.example.com Ready control-plane,master 8h v1.21.0 cka-2.example.com Ready <none> 173m v1.21.0 cka-3.example.com Ready <none> 172m v1.21.0 student@cka-1:~$ kubectl get pods -n kube-system NAME READY STATUS RESTARTS AGE calico-kube-controllers-594649bd75-dgs5z 1/1 Running 0 90s calico-node-72vk7 1/1 Running 0 90s calico-node-84tbc 1/1 Running 0 90s calico-node-qbh8q 1/1 Running 0 90s coredns-545d6fc579-pvc2w 1/1 Running 0 8h coredns-545d6fc579-q68xt 1/1 Running 0 8h etcd-cka-1.example.com 1/1 Running 1 8h kube-apiserver-cka-1.example.com 1/1 Running 1 8h kube-controller-manager-cka-1.example.com 1/1 Running 1 8h kube-proxy-c96sv 1/1 Running 1 174m kube-proxy-nv4mv 1/1 Running 1 173m kube-proxy-tpb6b 1/1 Running 1 8h kube-scheduler-cka-1.example.com 1/1 Running 1 8h
2.3补充
#配置kubectl命令自动补全 student@cka-1:~$ sudo apt install -y bash-completion student@cka-1:~$ source <(kubectl completion bash) #临时有效 ,重启无效 #将source <(kubectl completion bash) 当前用户的~/.bashrc student@cka-1:~$ echo "source <(kubectl completion bash)" >> ~/.bashrc 重启有效 配置客户端 #配置utility为kubernetes客户端: 配置centos kubernetes的仓库: [student@utility ~]$ sudo -i [root@utility ~]# cat <<EOF > /etc/yum.repos.d/kubernetes.repo > [kubernetes] > name=Kubernetes > baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ > enabled=1 > gpgcheck=0 > EOF [root@utility ~]# yum clean all [root@utility ~]# yum makecache #查找仓库中的kubectl 软件列表 [root@utility ~]# yum list --showduplicates kubectl --disableexcludes=kubernetes | grep '1.21.0' kubectl.x86_64 1.21.0-0 kubernetes #安装指定软件版本 [root@utility ~]# yum -y install kubectl-1.21.0-0 #kubectl连接集群 #创建 密钥目录:~/.kube [student@utility ~]$ mkdir ~/.kube [student@utility ~]$ scp student@cka-1:~/.kube/config ./.kube/ [student@utility ~]$ chmod 600 .kube/config [student@utility ~]$ source <(kubectl completion bash) [student@utility ~]$ echo "source <(kubectl completion bash)" >> ~/.bashrc [student@utility ~]$ kubectl get nodes NAME STATUS ROLES AGE VERSION cka-1.example.com Ready control-plane,master 9h v1.21.0 cka-2.example.com Ready <none> 3h36m v1.21.0 cka-3.example.com Ready <none> 3h35m v1.21.0