Ubuntu Desktop QEMU/KVM中使用Ubuntu Server 22.04配置k8s集群

Ubuntu Desktop QEMU/KVM中使用Ubuntu Server 22.04配置k8s集群

1.本机部署个代理

这里我是自己用docker+sing-box 部署的 http://192.168.2.105:10808

如果 用v2rayN等软件部署的，记得开启局域网共享

2.安装虚拟机

我本机使用的是Ubuntu Desktop ,这里在QEMU/KVM 中安装 Ubuntu Server 22.04

可以去官网下载：https://cn.ubuntu.com/download/server/step1

具体安装步骤参照：https://blog.csdn.net/u010080562/article/details/127708329

安装后拷贝三个供安装集群使用

3. 通用配置

1. 参照上边的教程 把ssh配置好
2. ssh远程链接
3. 配置hostname

hostnamectl set-hostname $host_name
hostname
sudo reboot

4. 配置网络

# 网关
gateway4=192.168.122.1
# 静态IP地址
ip=192.168.122.4
# 上边的根据自己实际情况改

apt update && apt install iputils-ping vim net-tools netplan.io wget -y
sudo tee /etc/netplan/01-network-manager-all.yaml > /dev/null <<EOF
network:
  version: 2
  renderer: networkd
  ethernets:
    enp1s0:
      addresses: [$ip/24]
      gateway4: $gateway4
      nameservers:
        addresses: [8.8.8.8]
EOF

chmod 600 /etc/netplan/01-network-manager-all.yaml
netplan apply

5. 配置防火墙

echo "配置防火墙"
sudo ufw allow 6443/tcp
sudo ufw allow 2379-2380/tcp
sudo ufw allow 10250-10259/tcp

6. 安装docker

echo "==> 卸载旧版本 Docker（如果存在）..."
sudo apt remove -y docker docker-engine docker.io containerd runc || true

echo "==> 安装依赖工具..."
sudo apt update
sudo apt install -y ca-certificates gnupg lsb-release

echo "==> 添加 Docker GPG 密钥..."
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | \
  sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg

echo "==> 添加阿里云 Docker 软件源..."
UBUNTU_CODENAME=$(lsb_release -cs)
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
  https://mirrors.aliyun.com/docker-ce/linux/ubuntu ${UBUNTU_CODENAME} stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

echo "==> 更新软件源并安装 Docker..."
sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin


echo "==> 启动并设置 Docker 开机自启..."
sudo systemctl enable docker
sudo systemctl start docker

echo "==> 配置国内镜像加速器（阿里云）..."
sudo mkdir -p /etc/docker
cat <<EOF | sudo tee /etc/docker/daemon.json
{
    "registry-mirrors": [
        "https://docker.m.daocloud.io",
        "https://docker.imgdb.de",
        "https://docker-0.unsee.tech",
        "https://docker.hlmirror.com",
        "https://docker.1ms.run",
        "https://func.ink",
        "https://lispy.org",
        "https://docker.xiaogenban1993.com"
    ]
}
EOF

# 配置拉取镜像的网络
sudo mkdir -p /etc/systemd/system/docker.service.d && \
echo -e \
"[Service]\nEnvironment=\"HTTP_PROXY=$proxy\"\nEnvironment=\"HTTPS_PROXY=$proxy\"\nEnvironment=\"NO_PROXY=localhost,127.0.0.1,::1,10.0.0.0/8,192.168.0.0/16\"" \
| sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf && \
sudo systemctl daemon-reexec && \
sudo systemctl daemon-reload && \
sudo systemctl restart docker && \
docker info | grep -i proxy


echo "==> 重新加载配置并重启 Docker..."
sudo systemctl daemon-reexec
sudo systemctl restart docker

echo " Docker 安装完成！版本信息如下："
docker --version

7. 安装k8s

echo "==> 设置 kubeadm 软件源"
sudo apt-get update
sudo apt install -y apt-transport-https ca-certificates
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

echo "==> 安装 kubelet、kubeadm 和 kubectl，并锁定其版本"
sudo apt-get update
sudo apt-get install -y kubelet=1.23.15-00 kubeadm=1.23.15-00 kubectl=1.23.15-00
sudo apt-mark hold kubelet kubeadm kubectl

# 加载 br_netfilter 内核模块
sudo modprobe br_netfilter
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1
echo "br_netfilter" | sudo tee /etc/modules-load.d/br_netfilter.conf
echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee /etc/sysctl.d/k8s.conf
echo "net.bridge.bridge-nf-call-ip6tables=1" | sudo tee -a /etc/sysctl.d/k8s.conf
sudo sysctl --system


sudo kubeadm reset -f
sudo rm -rf /etc/kubernetes/pki/  # 强制删除残留证书
sudo rm -rf /var/lib/kubelet/
sudo rm -f $HOME/.kube/config
sudo sed -i '/\/swap.img/ { /^[^#]/ s/^/#/ }' /etc/fstab
sudo swapoff -a
free -h

至此 通用配置完成

4. master配置

ip=192.168.122.4
# 加载 br_netfilter 内核模块
sudo modprobe br_netfilter
# 设置 sysctl 参数
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1
# 持久化配置（重启后生效）
echo "br_netfilter" | sudo tee /etc/modules-load.d/br_netfilter.conf
echo "net.bridge.bridge-nf-call-iptables=1" | sudo tee /etc/sysctl.d/k8s.conf
echo "net.bridge.bridge-nf-call-ip6tables=1" | sudo tee -a /etc/sysctl.d/k8s.conf
# 应用 sysctl 配置
sudo sysctl --system

# 使用阿里云镜像仓库拉取镜像
kubeadm config images pull \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.23.15
  
# 初始化集群
sudo kubeadm init \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.23.15 \
  --pod-network-cidr=10.244.0.0/16 \
  --apiserver-advertise-address=$ip \
  --ignore-preflight-errors=FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,SystemVerification
 
# 配置 kubectl 
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 安装 Pod 网络
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml

5. worker 添加到集群里

在master节点执行 命令 将结果粘贴到worker节点执行

kubeadm token create --print-join-command

如成功 则输出如图

如出错 重新执行 命令 清理缓存 处理问题 重新执行上边master输出的命令

# 清理之前的残留配置
sudo kubeadm reset -f
sudo rm -rf /etc/kubernetes/pki/  # 强制删除残留证书
sudo rm -rf /var/lib/kubelet/
sudo rm -f $HOME/.kube/config
sudo sed -i '/\/swap.img/ { /^[^#]/ s/^/#/ }' /etc/fstab
sudo swapoff -a
free -h

至此 所有安装配置完成，重复worker安装步骤在其他节点上即可

成果展示

常用命令记录：

# 查看集群节点
kubectl get nodes
# 删除集群节点
kubectl delete nodes 节点名
#  修改节点标签名
kubectl label node worker02 node-role.kubernetes.io/worker02=


shell脚本中加入 set -e 启动严格模式 碰到命令输出错误直接终止

问题记录：

1. KVM 虚拟机 网卡问题

sudo virsh net-list --all
sudo virsh net-start default
sudo virsh net-list --all

2. master not ready 问题处理 ----CNI 未正确启动（一般都是 kube-flannel 里的镜像拉取有问题）

# 配置代理
sudo mkdir -p /etc/systemd/system/docker.service.d && \
echo -e \
"[Service]\nEnvironment=\"HTTP_PROXY=http://192.168.2.105:10808\"\nEnvironment=\"HTTPS_PROXY=http://192.168.2.105:10808\"\nEnvironment=\"NO_PROXY=localhost,127.0.0.1,::1,10.0.0.0/8,192.168.0.0/16\"" \
| sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf && \
sudo systemctl daemon-reexec && \
sudo systemctl daemon-reload && \
sudo systemctl restart docker && \
docker info | grep -i proxy

# 安装 CNI
wget https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz && \
mv cni-plugins-linux-amd64-v0.8.6.tgz /opt/cni/bin/ && \
tar -zxvf cni-plugins-linux-amd64-v0.8.6.tgz

# 手动拉取镜像
docker pull ghcr.io/flannel-io/flannel:v0.26.7  && \
docker tag ghcr.io/flannel-io/flannel:v0.26.7 quay.io/coreos/flannel:v0.26.7 && \
docker images | grep flannel && \
wget https://raw.githubusercontent.com/flannel-io/flannel/v0.26.7/Documentation/kube-flannel.yml -O kube-flannel.yml

# vim进去修改
'''
containers:
  - name: kube-flannel
    image: quay.io/coreos/flannel:v0.26.7
'''
kubectl apply -f kube-flannel.yml && \
kubectl get pods -n kube-flannel -o wide && \
kubectl get nodes

检查代理网络是否可用 执行命令看输出即可

curl https://www.youtube.com/

代理不好使会一直卡住

代理好使会输出一堆html信息

第一次ssh链接 不知道ip可以在KVM中查看