Ubuntu(arm)部署k8s(kubernetes)集群

前言：

        k8s集群是目前高端运维需要掌握的必备技能之一，工作中你可以不用k8s，但是简历你不能没有k8s；面试造火箭，工作打螺丝；话不多说，直接上操作，这里就不过多阐述k8s的原理和作用了。

部署前工作

机器设备：MacBook Pro m1pro

虚拟系统：Ubuntu 22.04.3 LTS

Docker： v24.0.7 - 直接安装最新的即可
k8s： v1.26.2 - 根据最近两年的新版安装，旧版可能会存在很多bug

虚拟配置：

        稳妥起见，k8s至少需要2个CPU，4G运行

IP	CPU	内存
192.168.10.91	2	4G
192.168.10.91	2	4G
192.168.10.91	2	4G

# 配置ssh传输无密码传输，方便文件之间的传输

root@k8s-master-90:~# ssh-keygen -t rsa
root@k8s-master-91:~# ssh-keygen -t rsa
root@k8s-master-92:~# ssh-keygen -t rsa

root@k8s-master-90:~# ssh-copy-id 192.168.10.91
root@k8s-master-90:~# ssh-copy-id 192.168.10.92

root@k8s-master-91:~# ssh-copy-id 192.168.10.90
root@k8s-master-91:~# ssh-copy-id 192.168.10.92

root@k8s-master-92:~# ssh-copy-id 192.168.10.90
root@k8s-master-90:~# ssh-copy-id 192.168.10.91

系统上的基本配置

# 以下操作，都在每台主机上配置

1. 填写hosts域名映射

root@k8s-master-90:~# vim /etc/hosts

19.168.10.90   k8s-master-90
19.168.10.91   k8s-work-91
19.168.10.91   k8s-work-92

2. 关闭防火墙

root@k8s-master-90:~# systemctl status ufw.service
root@k8s-master-91:~# systemctl stop ufw.service
root@k8s-master-92:~# systemctl disable ufw.service

3、临时关闭关闭swap

root@k8s-master-90:~# systemctl status swap.target
root@k8s-master-91:~# systemctl stop swap.target
root@k8s-master-92:~# systemctl disable swap.target

4. 文件系统的静态信息的文件中/etc/fstab，注释掉swap

root@k8s-master-90:~# vim /etc/fstab
#/swap.img      none    swap    sw      0       0

5、修改内核，开启ipv4转发功能

root@k8s-master-90:~# tee /etc/sysctl.d/k8s.conf <<-'EOF'
net.ipv4.ip_forward = 1
EOF

6、查看是否开启转发功能成功，有输出即可

root@k8s-master-90:~# sysctl -p /etc/sysctl.d/k8s.conf 

7、安装ipvsadm

root@k8s-master-90:~#  apt-get install -y ipvsadm
root@k8s-master-90:~#  cat > /etc/modules-load.d/ipvs.conf << EOF
ip_vs_dh
ip_vs_fo
ip_vs_ftp 
ip_vs
ip_vs_lblc 
ip vs_lblcr 
ip_vs_lc
ip_vs_mh
ip_vs_nq
ip_vs_ovf
ip_vs_pe_sip 
ip_vs_rr
ip_vs_sed 
ip_vs_sh
ip_vs_wlc
ip_vs_wrr
nf_conntrack
EOF

8、开机自动systemd-modules-load.service

root@k8s-master-90:~#  systemctl enable --now systemd-modules-load.service

9、重启查看是否安装ipvsadm成功

root@k8s-master-90:~#  lsmod | grep ip_vs
root@k8s-master-90:~#  reboot
root@k8s-master-90:~#  lsmod | grep ip_vs

注意：以上操作，均需要在每台主机上操作！！

部署docker

同样的，这也需要在每台机器上安装docker，在这只演示一台主机安装流程

1、更新软件包

root@k8s-master-90:~#  apt-get update

2、安装依赖包

root@k8s-master-90:~#  apt-get install \
    ca-certificates \
    curl \
    gnupg \
    lsb-release

3、创建目录和docker下载文件  

root@k8s-master-90:~#  mkdir -p /etc/apt/keyrings

root@k8s-master-90:~#  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg    

 root@k8s-master-90:~#  echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 

4、安装docker-engine

root@k8s-master-90:~# apt-get update
 root@k8s-master-90:~# apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin

5、查看docker是否安装成功，并设置开机自启

root@k8s-master-90:~# docker images
root@k8s-master-90:~# systemctl status docker
root@k8s-master-90:~# systemctl enable docker
root@k8s-master-90:~# systemctl start docker
root@k8s-master-90:~# docker images 

6、配置docker镜像与cgroupdriver ，配置阿里云加速

root@k8s-master-90:~# cat > /etc/docker/daemon.json <<-EOF
{
    "registry-mirrors": [
        "https://docker.mirrors.ustc.edu.cn",
        "https://registry.docker-cn.com",
        "https://registry.docker-cn.com"
    ],
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m"
    },
    "storage-driver": "overlay2"
}
EOF

7、重置systemd，重启docker读取最新配置

root@k8s-master-90:~# systemctl daemon-reload
 root@k8s-master-90:~# systemctl restart docker
 root@k8s-master-90:~# docker images

安装cri-docker

k8s版本1.24以上就不在支持docker，需要手动安装

1、在GitHub下载二进制文件，cri-dockerd-0.3.8.arm64.tgz
链接：https://github.com/Mirantis/cri-dockerd/releases

2、解压文件

root@k8s-master-90:~# tar -xf cri-dockerd-0.3.8.arm64.tgz 

3、复制执行文件到/usr/bin

root@k8s-master-90:~/cri-dockerd# cp cri-dockerd /usr/bin/

4、创建文件cri-docker.socket 和cri-docker.service

root@k8s-master-90:/etc/systemd/system# vim cri-docker.service

[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

root@k8s-master-90:/etc/systemd/system# vim cri-docker.socket 

[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

5、重置systemd

root@k8s-master-90:/etc/systemd/system# systemctl daemon-reload 

6、产生systemd文件

root@k8s-master-90:/etc/systemd/system# systemctl enable cri-docker
Created symlink /etc/systemd/system/multi-user.target.wants/cri-docker.service → /etc/systemd/system/cri-docker.service.

7、启动cri-docker

root@k8s-master-90:/etc/systemd/system# systemctl start cri-docker

8、查看是否启动成功

root@k8s-master-90:/etc/systemd/system# systemctl status cri-docker

安装k8s(kubernetes)

1. 安装kubeadm、kubectl、kubelet
 阿里云教程链接：https://developer.aliyun.com/mirror/kubernetes?spm=a2c6h.13651102.0.0.3fac1b11P65H2N

此步骤1，需要在每台机器上安装

# kubectl 客户端工具
# kubeadm  管理集群程序
# kubelet  所有节点需要安装的 代理服务 

root@k8s-master-90:~#  apt-get update && apt-get install -y apt-transport-https
 root@k8s-master-90:~# curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - 
 root@k8s-master-90:~# cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
 root@k8s-master-90:~# apt-get update
 root@k8s-master-90:~# apt-get install kubelet=1.26.2-00 kubeadm=1.26.2-00 kubectl=1.26.2-00   # 可以根据版本进行安装，不指定默认最新版

# 启动kubelet 服务，并允许开机自启动

root@k8s-master-90:~# systemctl  start  kubelet 
root@k8s-master-90:~#​​​​​​​ systemctl enable  kubelet 
root@k8s-master-90:~# systemctl  status  kubelet

2. 在主节点设置master，初始化集群，此处使用90 当做master

        根据自己的IP，版本进行修改

root@k8s-master-90:~#  kubeadm init \
--image-repository  registry.aliyuncs.com/google_containers \
--kubernetes-version v1.26.2 --apiserver-advertise-address 192.168.10.90 \
--pod-network-cidr=10.244.0.0/16  \
--cri-socket unix:///var/run/cri-dockerd.sock \

3. 初始化完成后，会有要求配置，按照要求教程进行配置

        以及记住 kubeadm join 的那一条命令，node节点加入集群需要用到

root@k8s-master-90:~# mkdir -p $HOME/.kube
root@k8s-master-90:~# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
root@k8s-master-90:~# sudo chown $(id -u):$(id -g) $HOME/.kube/config

4. root用户操作的话，可以设置一个root 的环境变量

root@master-90 ~# export KUBECONFIG=/etc/kubernetes/admin.conf
root@master-90 ~# echo  "export KUBECONFIG=/etc/kubernetes/admin.conf"  >> /etc/profile

5. 初始化后，查看相关容器已经全部启动

root@master-90 ~# docker ps | awk '{print $NF}'
NAMES
k8s_kube-proxy_kube-proxy-hv8j9_kube-system_34a0e9d4-061e-4508-b270-17d43676b6eb_0
k8s_POD_kube-proxy-hv8j9_kube-system_34a0e9d4-061e-4508-b270-17d43676b6eb_0
k8s_kube-apiserver_kube-apiserver-k8s-master-90_kube-system_c50e991b430e64e8ec4726cad2f89510_0
k8s_etcd_etcd-k8s-master-90_kube-system_56b2f7c51f38056b60778da6aedd5aa2_0
k8s_kube-scheduler_kube-scheduler-k8s-master-90_kube-system_8981bc9d0b59c08f641d39fe6e9c0060_0
k8s_kube-controller-manager_kube-controller-manager-k8s-master-90_kube-system_26a10ba4e20f3c9dace25d2a4c9dfe10_0
k8s_POD_kube-apiserver-k8s-master-90_kube-system_c50e991b430e64e8ec4726cad2f89510_0
k8s_POD_etcd-k8s-master-90_kube-system_56b2f7c51f38056b60778da6aedd5aa2_0
k8s_POD_kube-controller-manager-k8s-master-90_kube-system_26a10ba4e20f3c9dace25d2a4c9dfe10_0
k8s_POD_kube-scheduler-k8s-master-90_kube-system_8981bc9d0b59c08f641d39fe6e9c0060_0

6. 默认没有tab键没有命令补齐，配置命令补齐

root@master-90 ~# kubectl  completion bash > /etc/bash_completion.d/kubelet
root@master-90 ~# kubeadm  completion bash > /etc/bash_completion.d/kubeadm
root@master-90 ~# exit   #重新登录，启动bash生效

7. 在master初始化后提供的命令，直接复制粘贴在node节点上运行，加入主集群

root@master-90 ~# kubeadm join 192.168.10.90:6443 --token r48gje.bbg8fns58hrkg2zz \
        --discovery-token-ca-cert-hash sha256:06a3255c64901c6ef61d1516dea897542b8f5a54441bd87ccd717c16038308d2   --cri-socket unix:///var/run/cri-dockerd.sock

    
8. 在主节点查看是否加入成功，此时 NotReady 是正常的

root@k8s-master-90:~# kubectl get nodes
NAME            STATUS     ROLES           AGE     VERSION
k8s-master-90   NotReady   control-plane   10m     v1.26.2
k8s-work-91     NotReady   <none>          4m33s   v1.26.2
k8s-work-92     NotReady   <none>          3m55s   v1.26.2

部署网络flannel

部署流程：

1. 下载文件：flannel-v0.24.0-linux-arm64.tar.gz
        下载地址：https://github.com/flannel-io/flannel/releases/

2. 下载kube-flannel.yml文件

root@k8s-master-90:~# mkdir  -p /root/flannel 
root@k8s-master-90:~/flannel# wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

3、查看要下载的镜像

root@k8s-master-90:~# grep image kube-flannel.yml 
        image: docker.io/flannel/flannel:v0.24.0
        image: docker.io/flannel/flannel-cni-plugin:v1.2.0
        image: docker.io/flannel/flannel:v0.24.0

4. 预先下载镜像      

 root@k8s-master-90:~# docker pull docker.io/flannel/flannel:v0.24.0
root@k8s-master-90:~# docker pull docker.io/flannel/flannel-cni-plugin:v1.2.0
root@k8s-master-90:~# docker pull docker.io/flannel/flannel:v0.24.0

5. node节点上也需要下载镜像

root@k8s-work-91:~# docker pull docker.io/flannel/flannel:v0.24.0
root@k8s-work-91:~# docker pull docker.io/flannel/flannel-cni-plugin:v1.2.0
root@k8s-work-91:~# docker pull docker.io/flannel/flannel:v0.24.0

root@k8s-work-92:~# docker pull docker.io/flannel/flannel:v0.24.0
root@k8s-work-92:~# docker pull docker.io/flannel/flannel-cni-plugin:v1.2.0
root@k8s-work-92:~# docker pull docker.io/flannel/flannel:v0.24.0

6. 应用yaml文件

root@k8s-master-90:~/flannel# kubectl apply -f kube-flannel.yml

7. 验证flannel应用是否成功

root@k8s-master-90:~/flannel# kubectl  get pods -n kube-flannel 
NAME                    READY   STATUS    RESTARTS   AGE
kube-flannel-ds-7x7rg   1/1     Running   0          39s
kube-flannel-ds-dwmtg   1/1     Running   0          39s
kube-flannel-ds-pqvdf   1/1     Running   0          39s

9. 查看节点是否OK，Ready说明可以连接成功

root@k8s-master-90:~/flannel# kubectl  get nodes
NAME            STATUS   ROLES           AGE    VERSION
k8s-master-90   Ready    control-plane   176m   v1.26.2
k8s-work-91     Ready    <none>          170m   v1.26.2
k8s-work-92     Ready    <none>          169m   v1.26.2
root@k8s-master-90:~/flannel# 

有一点需要注意：因为机器是arm架构的，关于arm的软件包有的非常少，目前还没有找到部署k8s可视化kuboard管理的软件包，官网也没有相关arm包，如果诸位前辈有部署的软件包，如果可以分享的话，十分感激！！

到此，k8s集群部署完成，后续还会支持更新！！！

        1. Docker实现CICD的流程

        2. k8s实现CICD的流程