openEuler-22.03-LTS-SP3二进制部署高可用Kubernetes

机器详情

主机名	IP地址	内存	CPU	硬盘
k8s-master01	192.168.46.31/24	4G	2C	50G
k8s-master02	192.168.46.32/24	4G	2C	50G
k8s-master03	192.168.46.33/24	4G	2C	50G
k8s-node01	192.168.46.34/24	4G	2C	50G
k8s-node02	192.168.46.35/24	4G	2C	50G

VIP地址	192.168.46.40/24
Pod网段	172.16.0.0/12
SVC网段	10.0.0.0/16

基本环境配置

设置主机名

hostnamectl set-hostname <主机名>

关闭防火墙

systemctl disable firewalld && systemctl stop firewalld

关闭selinux

vim /etc/selinux/config

# 修改SELINUX
SELINUX=disabled

配置hosts

vim /etc/hosts

192.168.46.31	k8s-master01
192.168.46.32	k8s-master02
192.168.46.33	k8s-master03
192.168.46.34	k8s-node01
192.168.46.35	k8s-node02

配置源为清华源

# 备份原有的源
cp /etc/yum.repos.d/openEuler.repo /etc/yum.repos.d/openEuler.repo.bak

sed -i "s#http:#https:#g" /etc/yum.repos.d/openEuler.repo

sed -i "s#repo.openeuler.org#mirrors.tuna.tsinghua.edu.cn\/openeuler#g" /etc/yum.repos.d/openEuler.repo

sed -i '/^meta/d' /etc/yum.repos.d/openEuler.repo

dnf clean all && dnf makecache && dnf -y update && dnf -y upgrade

配置时间同步

timedatectl set-timezone Asia/Shanghai

dnf -y install chrony

sed -i '/^pool/d' /etc/chrony.conf

echo "server ntp.aliyun.com iburst" >> /etc/chrony.conf

systemctl restart chronyd && systemctl enable chronyd

关闭Swap分区

swapoff -a && sysctl -w vm.swappiness=0

sed -i '/^[^#]*swap/s@^@#@' /etc/fstab

内核配置

cat <<EOF > /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
ip_vs_lblcr
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
net.ipv4.conf.all.route_localnet = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

dnf -y install ipvsadm ipset sysstat conntrack libseccomp

sysctl --system && reboot

k8s-master01免密其他节点

ssh-keygen -t rsa

for i in {
   1..5};do ssh-copy-id 192.168.46.3$i;done

部署Containerd（所有节点）

安装Containerd

wget https://github.com/containerd/containerd/releases/download/v1.7.18/cri-containerd-cni-1.7.18-linux-amd64.tar.gz

tar -zxvf cri-containerd-cni-1.7.18-linux-amd64.tar.gz -C /

配置内核

cat <<EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe -- overlay && modprobe -- br_netfilter

cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system

配置Containerd的配置文件

mkdir /etc/containerd

containerd config default > /etc/containerd/config.toml

sed -i "s#SystemdCgroup = false#SystemdCgroup = true#g" /etc/containerd/config.toml

sed -i 's#sandbox_image = "registry.k8s.io/pause:3.8"#sandbox_image = "swr.cn-north-4.myhuaweicloud.com/ctl456/registry-k8s-io-pause:3.8"#g' /etc/containerd/config.toml

配置crictl客户端

cat <<EOF > /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

启动containerd

systemctl daemon-reload && systemctl enable containerd && systemctl restart containerd

Kubernetes与Etcd部署

k8s-master01下载Kubernetes

wget https://dl.k8s.io/v1.30.2/kubernetes-server-linux-amd64.tar.gz

k8s-master01下载Etcd

wget https://github.com/etcd-io/etcd/releases/download/v3.5.14/etcd-v3.5.14-linux-amd64.tar.gz

k8s-master01-解压至bin目录

tar -zxvf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{
   let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

tar -zxvf etcd-v3.5.14-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.5.14-linux-amd64/etcd{
   ,ctl}

通过scp将组件发送到其他节点的bin目录

Kubernetes的master节点所需的组件

• kubelet
• kubectl
• kube-apiserver
• kube-controller-manager
• kube-scheduler
• kube-proxy
• etcd-ctl
• etcd

for i in {
   2..3};do scp /usr/local/bin/etcd* /usr/local/bin/kube* k8s-master0$i:/usr/local/bin/;done

Kubernetes的node节点所需的组件

• kubelet
• kube-proxy

for i in {
   1..2};do scp /usr/local/bin/kubelet /usr/local/bin/kube-proxy k8s-node0$i:/usr/local/bin/;done

生成证书

k8s-master01下载生成证书工具

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl_1.6.5_linux_amd64 -O /usr/local/bin/cfssl

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssljson_1.6.5_linux_amd64 -O /usr/local/bin/cfssljson

chmod +x /usr/local/bin/cfss*

生成Etcd证书

创建etcd证书目录（master节点）

mkdir -p /etc/etcd/ssl

创建kubernetes相关目录（所有节点）

mkdir -p /etc/kubernetes/pki

k8s-master01生成etcd证书

cat <<EOF > etcd-ca-csr.json
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca

cat <<EOF > etcd-csr.json
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOF

cat <<EOF > ca-config.json
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

cfssl gencert \
-ca=/etc/etcd/ssl/etcd-ca.pem \
-ca-key