1. 环境准备
1.1 k8s基础系统环境配置 （所有节点都要操作）
# 修改IPADDR 集群都要添加
vim /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=e4b85b5a-a925-45c3-ba75-5e74ac2151f0
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.206.6
NETMASK=255.255.255.0
GATEWAY=192.168.206.2
DNS1=8.8.8.8
# 设置主机名 (各个节点执行)
hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-master02
hostnamectl set-hostname k8s-master03
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02
# 所有节点配置hosts本地解析
cat > /etc/hosts <<EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.206.31 k8s-master01
192.168.206.32 k8s-master02
192.168.206.33 k8s-master03
192.168.206.34 k8s-node01
192.168.206.35 k8s-node02
192.168.206.99 lb-vip
EOF
1.2 上传文件到k8s-master01节点
# 根据具体情况具体处理 tar / zip 此处省略解压步骤
/root/centos
/root/kubernetes-v1.28.0
# 常用命令
yum install /root/centos7/* --skip-broken -y
sed -i 's/hostnamectl set-hostname k8s-master01/hostnamectl set-hostname k8s-master02/g' ../k8s-master02/initcentos.sh
hostls="k8s-master02 k8s-master03 k8s-node01 k8s-node02"
for wj in hostls;do cp initcentos.sh ../$hostls; done
export IP="192.168.206.31 192.168.206.33 192.168.206.32 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
node='k8s-master02 k8s-master03 k8s-node01 k8s-node02'
for NODE in $node; do scp -r install_containerd.sh $NODE: && chmod 777 install_containerd.sh && ./install_containerd.sh && mkdir -p /opt/cni/bin; done
1.2.1 初始化脚本
# 在master01
cd /root && mkdir install && cd install && vi initcentos.sh
#!/bin/bash
# 1、安装必要的工具
yum install /root/centos7/* --skip-broken -y
# 2、设置主机名
# hostnamectl set-hostname k8s-master01
# 3、关闭防火墙
systemctl disable --now firewalld
# 4、关闭SELinux
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
# 5、关闭交换分区
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a && sysctl -w vm.swappiness=0
cat /etc/fstab
# 6、网络配置
cat > /etc/NetworkManager/conf.d/calico.conf << EOF
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF
systemctl restart NetworkManager
# 7、进行时间同步
systemctl start chronyd
systemctl enable chronyd
chronyc sources -v
systemctl restart chronyd
# 8、配置ulimit
ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
soft nofile 655360
hard nofile 131072
soft nproc 655350
hard nproc 655350
seft memlock unlimited
hard memlock unlimitedd
EOF
# 9、安装ipvsadm
cat >> /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
systemctl restart systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack
# 10、修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
EOF
sysctl --system
# 11、配置环境变量
sed -i '1 i export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/opt/cni/bin' /etc/profile && source /etc/profile
# 在master01 执行initcentos.sh
chmod 777 initcentos.sh && ./initcentos.sh && cd /root
# 拷贝到在其他节点并执行initcentos.sh
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do scp -r centos7 install root@$HOST: ;done
for HOST in $IP;do ssh root@$HOST 'cd /root/install && chmod 777 initcentos.sh && ./initcentos.sh';done
1.3 配置免密登录
# master01
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
# master02
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
# master03
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
# node01
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.33 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
# node02
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.33 192.168.206.34"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
1.4 升级内核至4.18版本以上
# master01
yum install -y /root/kubernetes-v1.25.0/cby/kernel-lt-*-1.el7.elrepo.x86_64.rpm && grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) && grubby --default-kernel
# 其他节点
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do scp -r kubernetes-v1.25.0 $HOST: ;done
for HOST in $IP;do ssh $HOST 'yum install -y /root/kubernetes-v1.25.0/cby/kernel-lt-*-1.el7.elrepo.x86_64.rpm && grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) && grubby --default-kernel && reboot';done
# master01
reboot
2. k8s基本组件安装
2.1 安装Containerd作为Runtime （推荐）
#!/bin/bash
cd /root/kubernetes-v1.25.0/cby
mkdir -p /etc/cni/net.d /opt/cni/bin
tar xf cni-plugins-linux-amd64-v*.tgz -C /opt/cni/bin/
tar -xzf cri-containerd-cni-*-linux-amd64.tar.gz -C /
cat > /etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
systemctl restart systemd-modules-load.service
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep SystemdCgroup
sed -i "s#registry.k8s.io#m.daocloud.io/registry.k8s.io#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep sandbox_image
sed -i "s#config_path\ \=\ \"\"#config_path\ \=\ \"/etc/containerd/certs.d\"#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep certs.d
sed -i "s/k8s.gcr.io\/pause:3.6/registry.cn-hangzhou.aliyuncs.com\/chenby\/pause:3.6/g" /etc/containerd/config.toml
mkdir /etc/containerd/certs.d/docker.io -pv
cat > /etc/containerd/certs.d/docker.io/hosts.toml << EOF
server = "https://docker.io"
[host."https://hub-mirror.c.163.com"]
capabilities = ["pull", "resolve"]
EOF
systemctl daemon-reload
systemctl enable --now containerd.service
systemctl stop containerd.service
systemctl start containerd.service
systemctl restart containerd.service
systemctl status containerd.service
tar xf crictl-v*-linux-amd64.tar.gz -C /usr/bin/
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
systemctl restart containerd
crictl info
# master01执行安装脚本
cd /root/install && chmod 777 install-containerd.sh && ./install-containerd.sh
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'cd /root/install/ && chmod +X install-containerd.sh && ./install-containerd.sh';done
2.2 k8s与etcd下载及安装（仅在master01操作）
# 1、程序准备 在master01 执行
mkdir -p /opt/cni/bin
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /opt/cni/bin';done
cd /root/kubernetes-v1.25.0/cby
tar -xf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
tar -xf etcd*.tar.gz && mv etcd-*/etcd /usr/local/bin/ && mv etcd-*/etcdctl /usr/local/bin/
ls /usr/local/bin/
kubelet --version
etcdctl version
# 复制到其他节点
Master='k8s-master02 k8s-master03'
Work='k8s-node01 k8s-node02'
for NODE in $Master; do echo $NODE; scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done
for NODE in $Work; do echo $NODE; scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done
# cd /root/kubernetes-v1.25.0/cby
cp cfssl_1.6.2_linux_amd64 /usr/local/bin/cfssl
cp cfssljson_1.6.2_linux_amd64 /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson
## 2、生成各个组件证书
# etcd ssl 证书生成
mkdir /etc/etcd/ssl -p
/root/kubernetes-v1.25.0/tsl
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "876000h"
}
}
}
}
EOF
cat > etcd-ca-csr.json << EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF
cat > etcd-csr.json << EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
]
}
EOF
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca
cfssl gencert \
-ca=/etc/etcd/ssl/etcd-ca.pem \
-ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd
Master='k8s-master02 k8s-master03'
for NODE in $Master; do ssh $NODE "mkdir -p /etc/etcd/ssl &&"; for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem ; do scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}; done; done
mkdir -p /etc/kubernetes/pki
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /etc/kubernetes/pki && mkdir -p /data/k8s/etcd/';done
# # apiserver 证书生成
cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
],
"ca": {
"expiry": "876000h"
}
}
EOF
# apiserver 证书生成
cat > apiserver-csr.json << EOF
{
"CN": "kube-apiserver",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver
# front-proxy 证书生成
cat > front-proxy-ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"ca": {
"expiry": "876000h"
}
}
EOF
cat > front-proxy-client-csr.json << EOF
{
"CN": "front-proxy-client",
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca
cfssl gencert \
-ca=/etc/kubernetes/pki/front-proxy-ca.pem \
-ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem \
-config=ca-config.json \
-profile=kubernetes front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client
# controller-manage 证书生成
cat > manager-csr.json << EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-controller-manager",
"OU": "Kubernetes-manual"
}
]
}
EOF
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://127.0.0.1:8443 \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=/etc/kubernetes/pki/controller-manager.pem \
--client-key=/etc/kubernetes/pki/controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager@kubernetes \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# kube-scheduler 证书生成
cat > scheduler-csr.json << EOF
{
"CN": "system:kube-scheduler",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-scheduler",
"OU": "Kubernetes-manual"
}
]
}
EOF
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://127.0.0.1:8443 \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler \
--client-certificate=/etc/kubernetes/pki/scheduler.pem \
--client-key=/etc/kubernetes/pki/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
kubectl config use-context system:kube-scheduler@kubernetes \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# admin 证书生成
cat > admin-csr.json << EOF
{
"CN": "admin",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:masters",
"OU": "Kubernetes-manual"
}
]
}
EOF
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://127.0.0.1:8443 \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-credentials kubernetes-admin \
--client-certificate=/etc/kubernetes/pki/admin.pem \
--client-key=/etc/kubernetes/pki/admin-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-context kubernetes-admin@kubernetes \
--cluster=kubernetes \
--user=kubernetes-admin \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config use-context kubernetes-admin@kubernetes --kubeconfig=/etc/kubernetes/admin.kubeconfig
# kube-proxy证书生成
cat > kube-proxy-csr.json << EOF
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"k8s-master01",
"k8s-master02",
"k8s-master03",
"192.168.206.31",
"192.168.206.32",
"192.168.206.33",
"10.96.0.1",
"192.168.206.99",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"x.oiox.cn",
"k.oiox.cn",
"l.oiox.cn",
"o.oiox.cn",
"192.168.206.34",
"192.168.206.35",
"10.4.203.16",
"10.4.203.17",
"10.4.203.18",
"10.4.203.19",
"10.4.203.20"
],
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "system:kube-proxy",
"OU": "Kubernetes-manual"
}
]
}
EOF
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
kube-proxy-csr.json | cfssljson -bare /etc/kubernetes/pki/kube-proxy
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://127.0.0.1:8443 \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/pki/kube-proxy.pem \
--client-key=/etc/kubernetes/pki/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config set-context kube-proxy@kubernetes \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config use-context kube-proxy@kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub
2.3 查看证书
ls /etc/kubernetes/pki/
admin.csr controller-manager.csr kube-proxy.csr
admin-key.pem controller-manager-key.pem kube-proxy-key.pem
admin.pem controller-manager.pem kube-proxy.pem
apiserver.csr front-proxy-ca.csr sa.key
apiserver-key.pem front-proxy-ca-key.pem sa.pub
apiserver.pem front-proxy-ca.pem scheduler.csr
ca.csr front-proxy-client.csr scheduler-key.pem
ca-key.pem front-proxy-client-key.pem scheduler.pem
ca.pem front-proxy-client.pem
# 一共26个就对了
ls /etc/kubernetes/pki/ |wc -l
26
3. k8s系统组件安装
3.1 etcd配置、创建service、证书、启动、（所有master节点操作）
# # master01
mkdir -p /data/k8s/etcd/
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /data/k8s/etcd/';done
# master01
cat > /etc/etcd/etcd.config.yml << EOF
name: 'k8s-master01'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.31:2380'
listen-client-urls: 'https://192.168.206.31:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.31:2380'
advertise-client-urls: 'https://192.168.206.31:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service
# master02
cat > /etc/etcd/etcd.config.yml << EOF
name: 'k8s-master02'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.32:2380'
listen-client-urls: 'https://192.168.206.32:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.32:2380'
advertise-client-urls: 'https://192.168.206.32:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service
# master03
cat > /etc/etcd/etcd.config.yml << EOF
name: 'k8s-master03'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.33:2380'
listen-client-urls: 'https://192.168.206.33:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.33:2380'
advertise-client-urls: 'https://192.168.206.33:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service
3.2 查看etcd状态
# 如果要用IPv6那么把IPv4地址修改为IPv6即可
export ETCDCTL_API=3
etcdctl --endpoints="192.168.206.31:2379,192.168.206.32:2379,192.168.206.33:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem endpoint status --write-out=table
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 192.168.206.33:2379 | 6ae2196f75cd6d95 | 3.5.9 | 20 kB | false | false | 2 | 9 | 9 | |
| 192.168.206.32:2379 | 46cbf93f7713a252 | 3.5.9 | 20 kB | false | false | 2 | 9 | 9 | |
| 192.168.206.31:2379 | ec6051ffc7487dd7 | 3.5.9 | 20 kB | true | false | 2 | 9 | 9 | |
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
etcdctl endpoint health --endpoints="192.168.206.31:2379,192.168.206.32:2379,192.168.206.33:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem
4. 高可用配置（在Master服务器上操作）
cd /root/kubernetes-v1.25.0/cby/ && tar xvf nginx-*.tar.gz && cd nginx-* ./configure --with-stream --without-http --without-http_uwsgi_module --without-http_scgi_module --without-http_fastcgi_module &&/
make && make install
4.1 NGINX高可用方案
4.1.1 进行编译
# 安装编译环境
cd
yum install gcc -y
tar xvf nginx-*.tar.gz
cd nginx-*
# 进行编译
./configure --with-stream --without-http --without-http_uwsgi_module --without-http_scgi_module --without-http_fastcgi_module
make && make install
# 拷贝编译好的nginx
node='k8s-master02 k8s-master03 k8s-node01 k8s-node02'
for NODE in $node; do scp -r /usr/local/nginx/ $NODE:/usr/local/nginx/; done
# 这是一系列命令行指令，用于编译和安装软件。
#
# 1. `./configure` 是用于配置软件的命令。在这个例子中，配置的软件是一个Web服务器，指定了一些选项来启用流模块，并禁用了HTTP、uwsgi、scgi和fastcgi模块。
# 2. `--with-stream` 指定启用流模块。流模块通常用于代理TCP和UDP流量。
# 3. `--without-http` 指定禁用HTTP模块。这意味着编译的软件将没有HTTP服务器功能。
# 4. `--without-http_uwsgi_module` 指定禁用uwsgi模块。uwsgi是一种Web服务器和应用服务器之间的通信协议。
# 5. `--without-http_scgi_module` 指定禁用scgi模块。scgi是一种用于将Web服务器请求传递到应用服务器的协议。
# 6. `--without-http_fastcgi_module` 指定禁用fastcgi模块。fastcgi是一种用于在Web服务器和应用服务器之间交换数据的协议。
# 7. `make` 是用于编译软件的命令。该命令将根据之前的配置生成可执行文件。
# 8. `make install` 用于安装软件。该命令将生成的可执行文件和其他必要文件复制到系统的适当位置，以便可以使用该软件。
#
# 总之，这个命令序列用于编译一个配置了特定选项的Web服务器，并将其安装到系统中。
4.1.2 写入启动配置
在所有主机上执行
# 写入nginx配置文件
cat > /usr/local/nginx/conf/kube-nginx.conf <<EOF
worker_processes 1;
events {
worker_connections 1024;
}
stream {
upstream backend {
least_conn;
hash $remote_addr consistent;
server 192.168.206.31:6443 max_fails=3 fail_timeout=30s;
server 192.168.206.32:6443 max_fails=3 fail_timeout=30s;
server 192.168.206.33:6443 max_fails=3 fail_timeout=30s;
}
server {
listen 127.0.0.1:8443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
EOF
# 这段配置是一个nginx的stream模块的配置，用于代理TCP和UDP流量。
#
# 首先，`worker_processes 1;`表示启动一个worker进程用于处理流量。
# 接下来，`events { worker_connections 1024; }`表示每个worker进程可以同时处理最多1024个连接。
# 在stream块里面，定义了一个名为`backend`的upstream，用于负载均衡和故障转移。
# `least_conn`表示使用最少连接算法进行负载均衡。
# `hash $remote_addr consistent`表示用客户端的IP地址进行哈希分配请求，保持相同IP的请求始终访问同一台服务器。
# `server`指令用于定义后端的服务器，每个服务器都有一个IP地址和端口号，以及一些可选的参数。
# `max_fails=3`表示当一个服务器连续失败3次时将其标记为不可用。
# `fail_timeout=30s`表示如果一个服务器被标记为不可用，nginx将在30秒后重新尝试。
# 在server块内部，定义了一个监听地址为127.0.0.1:8443的服务器。
# `proxy_connect_timeout 1s`表示与后端服务器建立连接的超时时间为1秒。
# `proxy_pass backend`表示将流量代理到名为backend的上游服务器组。
#
# 总结起来，这段配置将流量代理到一个包含3个后端服务器的上游服务器组中，使用最少连接算法进行负载均衡，并根据客户端的IP地址进行哈希分配请求。如果一个服务器连续失败3次，则将其标记为不可用，并在30秒后重新尝试。
# 写入启动配置文件
cat > /etc/systemd/system/kube-nginx.service <<EOF
[Unit]
Description=kube-apiserver nginx proxy
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
ExecStartPre=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx -t
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx
ExecReload=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx -s reload
PrivateTmp=true
Restart=always
RestartSec=5
StartLimitInterval=0
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 这是一个用于kube-apiserver的NGINX代理的systemd单位文件。
#
# [Unit]部分包含了单位的描述和依赖关系。它指定了在network.target和network-online.target之后启动，并且需要network-online.target。
#
# [Service]部分定义了如何运行该服务。Type指定了服务进程的类型（forking表示主进程会派生一个子进程）。ExecStartPre指定了在服务启动之前需要运行的命令，用于检查NGINX配置文件的语法是否正确。ExecStart指定了启动服务所需的命令。ExecReload指定了在重新加载配置文件时运行的命令。PrivateTmp设置为true表示将为服务创建一个私有的临时文件系统。Restart和RestartSec用于设置服务的自动重启机制。StartLimitInterval设置为0表示无需等待，可以立即重启服务。LimitNOFILE指定了服务的文件描述符的限制。
#
# [Install]部分指定了在哪些target下该单位应该被启用。
#
# 综上所述，此单位文件用于启动和管理kube-apiserver的NGINX代理服务。它通过NGINX来反向代理和负载均衡kube-apiserver的请求。该服务会在系统启动时自动启动，并具有自动重启的机制。
# 设置开机自启
systemctl daemon-reload
systemctl enable --now kube-nginx.service
systemctl restart kube-nginx.service
systemctl status kube-nginx.service
## 5.2 keepalived和haproxy 高可用方案
### 5.2.1安装keepalived和haproxy服务
systemctl disable --now firewalld
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
yum -y install keepalived haproxy
4.1.3 修改haproxy配置文件（配置文件一样）
# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak
cat >/etc/haproxy/haproxy.cfg<<"EOF"
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor
frontend k8s-master
bind 0.0.0.0:9443
bind 127.0.0.1:9443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-master01 192.168.206.31:6443 check
server k8s-master02 192.168.206.32:6443 check
server k8s-master03 192.168.206.33:6443 check
EOF
参数
这段配置代码是指定了一个HAProxy负载均衡器的配置。下面对各部分进行详细解释：
1. global:
- maxconn 2000: 设置每个进程的最大连接数为2000。
- ulimit-n 16384: 设置每个进程的最大文件描述符数为16384。
- log 127.0.0.1 local0 err: 指定日志的输出地址为本地主机的127.0.0.1，并且只记录错误级别的日志。
- stats timeout 30s: 设置查看负载均衡器统计信息的超时时间为30秒。
2. defaults:
- log global: 使默认日志与global部分相同。
- mode http: 设定负载均衡器的工作模式为HTTP模式。
- option httplog: 使负载均衡器记录HTTP协议的日志。
- timeout connect 5000: 设置与后端服务器建立连接的超时时间为5秒。
- timeout client 50000: 设置与客户端的连接超时时间为50秒。
- timeout server 50000: 设置与后端服务器连接的超时时间为50秒。
- timeout http-request 15s: 设置处理HTTP请求的超时时间为15秒。
- timeout http-keep-alive 15s: 设置保持HTTP连接的超时时间为15秒。
3. frontend monitor-in:
- bind *:33305: 监听所有IP地址的33305端口。
- mode http: 设定frontend的工作模式为HTTP模式。
- option httplog: 记录HTTP协议的日志。
- monitor-uri /monitor: 设置监控URI为/monitor。
4. frontend k8s-master:
- bind 0.0.0.0:9443: 监听所有IP地址的9443端口。
- bind 127.0.0.1:9443: 监听本地主机的9443端口。
- mode tcp: 设定frontend的工作模式为TCP模式。
- option tcplog: 记录TCP协议的日志。
- tcp-request inspect-delay 5s: 设置在接收到请求后延迟5秒进行检查。
- default_backend k8s-master: 设置默认的后端服务器组为k8s-master。
5. backend k8s-master:
- mode tcp: 设定backend的工作模式为TCP模式。
- option tcplog: 记录TCP协议的日志。
- option tcp-check: 启用TCP检查功能。
- balance roundrobin: 使用轮询算法进行负载均衡。
- default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100: 设置默认的服务器参数。
- server k8s-master01 192.168.206.31:6443 check: 增加一个名为k8s-master01的服务器，IP地址为192.168.206.31，端口号为6443，并对其进行健康检查。
- server k8s-master02 192.168.206.32:6443 check: 增加一个名为k8s-master02的服务器，IP地址为192.168.206.32，端口号为6443，并对其进行健康检查。
- server k8s-master03 192.168.206.33:6443 check: 增加一个名为k8s-master03的服务器，IP地址为192.168.206.33，端口号为6443，并对其进行健康检查。
以上就是这段配置代码的详细解释。它主要定义了全局配置、默认配置、前端监听和后端服务器组的相关参数和设置。通过这些配置，可以实现负载均衡和监控功能。
4.1.4 Master01配置keepalived master节点
#cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
# 注意网卡名
interface ens33
mcast_src_ip 192.168.206.31
virtual_router_id 51
priority 100
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.206.99
}
track_script {
chk_apiserver
} }
EOF
4.1.5 Master02配置keepalived backup节点
# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
# 注意网卡名
interface eth0
mcast_src_ip 192.168.206.32
virtual_router_id 51
priority 80
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.206.99
}
track_script {
chk_apiserver
} }
EOF
4.1.6 Master03配置keepalived backup节点
# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
# 注意网卡名
interface eth0
mcast_src_ip 192.168.206.33
virtual_router_id 51
priority 50
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.206.99
}
track_script {
chk_apiserver
} }
EOF
参数
这是一个用于配置keepalived的配置文件。下面是对每个部分的详细解释：
- `global_defs`部分定义了全局参数。
- `router_id`参数指定了当前路由器的标识，这里设置为"LVS_DEVEL"。
- `vrrp_script`部分定义了一个VRRP脚本。`chk_apiserver`是脚本的名称，
- `script`参数指定了脚本的路径。该脚本每5秒执行一次，返回值为0表示服务正常，返回值为1表示服务异常。
- `weight`参数指定了根据脚本返回的值来调整优先级，这里设置为-5。
- `fall`参数指定了失败阈值，当连续2次脚本返回值为1时认为服务异常。
- `rise`参数指定了恢复阈值，当连续1次脚本返回值为0时认为服务恢复正常。
- `vrrp_instance`部分定义了一个VRRP实例。`VI_1`是实例的名称。
- `state`参数指定了当前实例的状态，这里设置为MASTER表示当前实例是主节点。
- `interface`参数指定了要监听的网卡，这里设置为eth0。
- `mcast_src_ip`参数指定了VRRP报文的源IP地址，这里设置为192.168.206.31。
- `virtual_router_id`参数指定了虚拟路由器的ID，这里设置为51。
- `priority`参数指定了实例的优先级，优先级越高（数值越大）越有可能被选为主节点。
- `nopreempt`参数指定了当主节点失效后不要抢占身份，即不要自动切换为主节点。
- `advert_int`参数指定了发送广播的间隔时间，这里设置为2秒。
- `authentication`部分指定了认证参数
- `auth_type`参数指定了认证类型，这里设置为PASS表示使用密码认证，
- `auth_pass`参数指定了认证密码，这里设置为K8SHA_KA_AUTH。
- `virtual_ipaddress`部分指定了虚拟IP地址，这里设置为192.168.206.99。
- `track_script`部分指定了要跟踪的脚本，这里跟踪了chk_apiserver脚本。
4.1.7 健康检查脚本配置（lb主机）
cat > /etc/keepalived/check_apiserver.sh << EOF
#!/bin/bash
err=0
for k in \$(seq 1 3)
do
check_code=\$(pgrep haproxy)
if [[ \$check_code == "" ]]; then
err=\$(expr \$err + 1)
sleep 1
continue
else
err=0
break
fi
done
if [[ \$err != "0" ]]; then
echo "systemctl stop keepalived"
/usr/bin/systemctl stop keepalived
exit 1
else
exit 0
fi
EOF
# 给脚本授权
chmod +x /etc/keepalived/check_apiserver.sh
# 这段脚本是一个简单的bash脚本，主要用来检查是否有名为haproxy的进程正在运行。
#
# 脚本的主要逻辑如下：
# 1. 首先设置一个变量err为0，用来记录错误次数。
# 2. 使用一个循环，在循环内部执行以下操作：
# a. 使用pgrep命令检查是否有名为haproxy的进程在运行。如果不存在该进程，将err加1，并暂停1秒钟，然后继续下一次循环。
# b. 如果存在haproxy进程，将err重置为0，并跳出循环。
# 3. 检查err的值，如果不为0，表示检查失败，输出一条错误信息并执行“systemctl stop keepalived”命令停止keepalived进程，并退出脚本返回1。
# 4. 如果err的值为0，表示检查成功，退出脚本返回0。
#
# 该脚本的主要作用是检查是否存在运行中的haproxy进程，如果无法检测到haproxy进程，将停止keepalived进程并返回错误状态。如果haproxy进程存在，则返回成功状态。这个脚本可能是作为一个健康检查脚本的一部分，在确保haproxy服务可用的情况下，才继续运行其他操作。
4.1.8 启动服务
systemctl daemon-reload
systemctl enable --now haproxy.service
systemctl enable --now keepalived.service
systemctl status haproxy.service
systemctl status keepalived.service
# keepalived.service单元的当前状态，包括运行状态、是否启用等信息。
4.1.9 测试高可用
# 能ping同
[root@k8s-node02 ~]# ping 192.168.206.99
# 能telnet访问
[root@k8s-node02 ~]# telnet 192.168.206.99 9443
# 关闭主节点，看vip是否漂移到备节点
5. k8s组件配置
所有k8s节点创建以下目录
mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes
export IP = "192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes';done
5.1 创建并启动apiserver（所有master节点）
5.1.1 master01节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
--v=2 \\
--allow-privileged=true \\
--bind-address=0.0.0.0 \\
--secure-port=6443 \\
--advertise-address=192.168.206.31 \\
--service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112 \\
--service-node-port-range=30000-32767 \\
--etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
--etcd-certfile=/etc/etcd/ssl/etcd.pem \\
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/sa.pub \\
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \\
--enable-bootstrap-token-auth=true \\
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
--requestheader-allowed-names=aggregator \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-extra-headers-prefix=X-Remote-Extra- \\
--requestheader-username-headers=X-Remote-User \\
--enable-aggregator-routing=true
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
5.1.2 master02节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
--v=2 \\
--allow-privileged=true \\
--bind-address=0.0.0.0 \\
--secure-port=6443 \\
--advertise-address=192.168.206.32 \\
--service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112 \\
--service-node-port-range=30000-32767 \\
--etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
--etcd-certfile=/etc/etcd/ssl/etcd.pem \\
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/sa.pub \\
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\
--authorization-mode=Node,RBAC \\
--enable-bootstrap-token-auth=true \\
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
--requestheader-allowed-names=aggregator \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-extra-headers-prefix=X-Remote-Extra- \\
--requestheader-username-headers=X-Remote-User \\
--enable-aggregator-routing=true
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
5.1.3 master03节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
--v=2 \\
--allow-privileged=true \\
--bind-address=0.0.0.0 \\
--secure-port=6443 \\
--advertise-address=192.168.206.33 \\
--service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112 \\
--service-node-port-range=30000-32767 \\
--etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \\
--etcd-certfile=/etc/etcd/ssl/etcd.pem \\
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \\
--client-ca-file=/etc/kubernetes/pki/ca.pem \\
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
--service-account-key-file=/etc/kubernetes/pki/sa.pub \\
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \\
--service-account-issuer=https://kubernetes.default.svc.cluster.local \\
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \\
--authorization-mode=Node,RBAC \\
--enable-bootstrap-token-auth=true \\
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \\
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \\
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \\
--requestheader-allowed-names=aggregator \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-extra-headers-prefix=X-Remote-Extra- \\
--requestheader-username-headers=X-Remote-User \\
--enable-aggregator-routing=true
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now kube-apiserver.service
systemctl restart kube-apiserver.service
systemctl status kube-apiserver.service
5.2 启动kube-controller-manager，并查看状态
## 6.2.配置kube-controller-manager service
```shell
# 所有master节点配置，且配置相同
# 172.16.0.0/12为pod网段，按需求设置你自己的网段
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
--v=2 \\
--bind-address=0.0.0.0 \\
--root-ca-file=/etc/kubernetes/pki/ca.pem \\
--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
--service-account-private-key-file=/etc/kubernetes/pki/sa.key \\
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\
--leader-elect=true \\
--use-service-account-credentials=true \\
--node-monitor-grace-period=40s \\
--node-monitor-period=5s \\
--controllers=*,bootstrapsigner,tokencleaner \\
--allocate-node-cidrs=true \\
--service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112 \\
--cluster-cidr=172.16.0.0/12,fc00:2222::/112 \\
--node-cidr-mask-size-ipv4=24 \\
--node-cidr-mask-size-ipv6=120 \\
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now kube-controller-manager.service
systemctl restart kube-controller-manager.service
systemctl status kube-controller-manager.service
5.3 配置kube-scheduler service
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-scheduler \\
--v=2 \\
--bind-address=0.0.0.0 \\
--leader-elect=true \\
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now kube-scheduler.service
systemctl restart kube-scheduler.service
systemctl status kube-scheduler.service
6. TLS Bootstrapping配置
6.1 在master01上配置
cd /root/kubernetes-v1.25.0/bootstrap
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true --server=https://127.0.0.1:8443 \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-credentials tls-bootstrap-token-user \
--token=c8ad9c.2e4d610cf3e7426e \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-context tls-bootstrap-token-user@kubernetes \
--cluster=kubernetes \
--user=tls-bootstrap-token-user \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config use-context tls-bootstrap-token-user@kubernetes \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config
6.2 查看集群状态，没问题的话继续后续操作
kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {"health":"true","reason":""}
etcd-2 Healthy {"health":"true","reason":""}
etcd-1 Healthy {"health":"true","reason":""}
# 切记执行，别忘记！！！
kubectl create -f bootstrap.secret.yaml
7. node节点配置
7.1 在master01上将证书复制到node节点
cd /etc/kubernetes/
for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do ssh $NODE mkdir -p /etc/kubernetes/pki; for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig kube-proxy.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}; done; done
for NODE in k8s-master02 k8s-master03; do for FILE in $(ls /etc/kubernetes/pki | grep -v etcd); do scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE}; done; for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig; do scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE}; done; done
7.2 kubelet配置
mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/
# 所有k8s节点配置kubelet service
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service
[Service]
ExecStart=/usr/local/bin/kubelet \\
--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig \\
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
--config=/etc/kubernetes/kubelet-conf.yml \\
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \\
--node-labels=node.kubernetes.io/node=
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/kubernetes/kubelet-conf.yml <<EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
systemctl daemon-reload
systemctl enable --now kubelet.service
systemctl restart kubelet.service
systemctl status kubelet.service
### 8.2.5查看集群
```shell
[root@k8s-master01 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready <none> 65s v1.25.0
k8s-master02 Ready <none> 49s v1.25.0
k8s-master03 Ready <none> 38s v1.25.0
k8s-node01 Ready <none> 33s v1.25.0
k8s-node02 Ready <none> 30s v1.25.0
# 查看容器运行时
[root@k8s-master01 ~]# kubectl describe node | grep Runtime
Container Runtime Version: containerd://1.6.8
Container Runtime Version: containerd://1.6.8
Container Runtime Version: containerd://1.6.8
Container Runtime Version: containerd://1.6.8
Container Runtime Version: containerd://1.6.8
7.3 所有k8s节点添加kube-proxy
# master-1执行
for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig; done
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-proxy \\
--config=/etc/kubernetes/kube-proxy.yaml \\
--v=2
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/kubernetes/kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
qps: 5
clusterCIDR: 172.16.0.0/12,fc00:2222::/112
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
ipvs:
masqueradeAll: true
minSyncPeriod: 5s
scheduler: "rr"
syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF
systemctl daemon-reload
systemctl enable --now kube-proxy.service
systemctl restart kube-proxy.service
systemctl status kube-proxy.service
7.4 安装Calico
7.4.1 更改calico网段
cd /root/kubernetes-v1.25.0/
# 本地没有公网 IPv6 使用 calico.yaml
kubectl apply -f calico.yaml
# 本地有公网 IPv6 使用 calico-ipv6.yaml
# kubectl apply -f calico-ipv6.yaml
7.4.2 查看容器状态
# calico 初始化会很慢 需要耐心等待一下，大约十分钟左右
[root@k8s-master01 ~]# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-6747f75cdc-fbvvc 1/1 Running 0 61s
kube-system calico-node-fs7hl 1/1 Running 0 61s
kube-system calico-node-jqz58 1/1 Running 0 61s
kube-system calico-node-khjlg 1/1 Running 0 61s
kube-system calico-node-wmf8q 1/1 Running 0 61s
kube-system calico-node-xc6gn 1/1 Running 0 61s
kube-system calico-typha-6cdc4b4fbc-57snb 1/1 Running 0 61s
8. 安装helm
cd /root/kubernetes-v1.25.0/cby/
tar xvf helm-*-linux-amd64.tar.gz
cp linux-amd64/helm /usr/local/bin/
9. 安装CoreDNS
# master01操作
cd /root/kubernetes-v1.25.0/calico/
kubectl apply -f calico.yaml
10. 安装Metrics Server
cd /root/kubernetes-v1.25.0/metrics-server/
kubectl apply -f metrics-server.yaml
# kubectl apply -f high-availability.yaml
# 稍等片刻查看状态
kubectl top node
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
k8s-master01 197m 4% 1497Mi 39%
k8s-master02 152m 3% 1315Mi 34%
k8s-master03 112m 2% 1274Mi 33%
k8s-node01 142m 3% 777Mi 20%
k8s-node02 71m 1% 682Mi 17%
11. 集群验证
11.1 部署pod资源
cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: busybox
namespace: default
spec:
containers:
- name: busybox
image: docker.io/library/busybox:1.28
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
restartPolicy: Always
EOF
# 查看
kubectl get pod
NAME READY STATUS RESTARTS AGE
busybox 1/1 Running 0 17s
11.2 用pod解析默认命名空间中的kubernetes
# 查看name
kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 17h
# 进行解析
kubectl exec busybox -n default -- nslookup kubernetes
3Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
11.3 测试跨命名空间是否可以解析
# 查看有那些name
kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 76m
kube-system calico-typha ClusterIP 10.105.100.82 <none> 5473/TCP 35m
kube-system coredns-coredns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP 8m14s
kube-system metrics-server ClusterIP 10.105.60.31 <none> 443/TCP 109s
# 进行解析
kubectl exec busybox -n default -- nslookup coredns-coredns.kube-system
Server: 10.96.0.10
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local
Name: coredns-coredns.kube-system
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local
[root@k8s-master01 metrics-server]#
11.4 每个节点都必须要能访问Kubernetes的kubernetes svc 443和kube-dns的service 53
telnet 10.96.0.1 443
Trying 10.96.0.1...
Connected to 10.96.0.1.
Escape character is '^]'.
telnet 10.96.0.10 53
Trying 10.96.0.10...
Connected to 10.96.0.10.
Escape character is '^]'.
curl 10.96.0.10:53
curl: (52) Empty reply from server
11.5 Pod和Pod之前要能通
kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 0 17m 172.27.14.193 k8s-node02 <none> <none>
kubectl get po -n kube-system -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
calico-kube-controllers-76754ff848-pw4xg 1/1 Running 0 38m 172.25.244.193 k8s-master01 <none> <none>
calico-node-97m55 1/1 Running 0 38m 192.168.0.34 k8s-node01 <none> <none>
calico-node-hlz7j 1/1 Running 0 38m 192.168.206.32 k8s-master02 <none> <none>
calico-node-jtlck 1/1 Running 0 38m 192.168.206.33 k8s-master03 <none> <none>
calico-node-lxfkf 1/1 Running 0 38m 192.168.0.35 k8s-node02 <none> <none>
calico-node-t667x 1/1 Running 0 38m 192.168.206.31 k8s-master01 <none> <none>
calico-typha-59d75c5dd4-gbhfp 1/1 Running 0 38m 192.168.0.35 k8s-node02 <none> <none>
coredns-coredns-c5c6d4d9b-bd829 1/1 Running 0 10m 172.25.92.65 k8s-master02 <none> <none>
metrics-server-7c8b55c754-w7q8v 1/1 Running 0 3m56s 172.17.125.3 k8s-node01 <none> <none>
# 进入busybox ping其他节点上的pod
kubectl exec -ti busybox -- sh
/ # ping 192.168.0.34
PING 192.168.0.34 (192.168.0.34): 56 data bytes
64 bytes from 192.168.0.34: seq=0 ttl=63 time=0.358 ms
64 bytes from 192.168.0.34: seq=1 ttl=63 time=0.668 ms
64 bytes from 192.168.0.34: seq=2 ttl=63 time=0.637 ms
64 bytes from 192.168.0.34: seq=3 ttl=63 time=0.624 ms
64 bytes from 192.168.0.34: seq=4 ttl=63 time=0.907 ms
# 可以连通证明这个pod是可以跨命名空间和跨主机通信的
11.6 创建三个副本，可以看到3个副本分布在不同的节点上（用完可以删了）
cat > deployments.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: docker.io/library/nginx:1.25.3
ports:
- containerPort: 80
EOF
kubectl apply -f deployments.yaml
deployment.apps/nginx-deployment created
kubectl get pod
NAME READY STATUS RESTARTS AGE
busybox 1/1 Running 0 6m25s
nginx-deployment-9456bbbf9-4bmvk 1/1 Running 0 8s
nginx-deployment-9456bbbf9-9rcdk 1/1 Running 0 8s
nginx-deployment-9456bbbf9-dqv8s 1/1 Running 0 8s
# 删除nginx
[root@k8s-master01 ~]# kubectl delete -f deployments.yaml
12. 安装dashboard
12.1 更改dashboard的svc为NodePort，如果已是请忽略
kubectl edit svc kubernetes-dashboard -n kube-system
type: NodePort
12.2 查看端口号
kubectl get svc kubernetes-dashboard -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard NodePort 10.108.120.110 <none> 443:30034/TCP 34s
12.3 创建token
cat > dashboard-user.yaml << EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kube-system
EOF
kubectl apply -f dashboard-user.yaml
# 创建token
kubectl -n kube-system create token admin-user
eyJhbGciOiJSUzI1NiIsImtpZCI6IksxY2U2U19KUWlRMzJSVXdtU2wzak1PdXpJYXVxQTBlbGJHUWlQZWN0ZU0ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjkyNDQ3NzA3LCJpYXQiOjE2OTI0NDQxMDcsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiNmE4MWEwY2ItM2U0Yi00ZTNhLTk0N2EtY2ViNDNkOTNjZmUzIn19LCJuYmYiOjE2OTI0NDQxMDcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbi11c2VyIn0.Ww8zpmguHtxAuUn1EWtNCP2A-d25PGOYO3_FkHyOtj6f0iLm_HTvwM0InlTgIAWnfWMDOHzBLc9m1gYzoaC5efgBVtZkpy900NIhW_-yiQK3cMpiNasKOH7jiPnNMXNXczw3ElZWMqFYXkYRmQRVgVd6t0DmYK_TCXjDiZIU9jCzIDdSWDDI9nIieRGQwY8CzfEM9CKeYYC4a5wOG6t4ZuTcnRAYdZ1KZ7PZ1R73JLauessAtiDUArTIB2xWcWxy_b_J4-wXtsQyW5YOYOQ3Ie9NbERQj9wlprNSLhFqSxq-RUwizGBZ7z7t1RmW134DStU25uA4GkSJBQWK4b1cWA
12.4 登录dashboard
https://192.168.206.31:30034/
13. ingress安装
13.1 执行部署
wget https://mirrors.chenby.cn/https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/cloud/deploy.yaml
# 修改为国内源 docker源可选
sed -i "s#registry.k8s.io/#m.daocloud.io/registry.k8s.io/#g" *.yaml
cat > backend.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: default-http-backend
labels:
app.kubernetes.io/name: default-http-backend
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: default-http-backend
template:
metadata:
labels:
app.kubernetes.io/name: default-http-backend
spec:
terminationGracePeriodSeconds: 60
containers:
- name: default-http-backend
image: registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5
livenessProbe:
httpGet:
path: /healthz
port: 8080
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 5
ports:
- containerPort: 8080
resources:
limits:
cpu: 10m
memory: 20Mi
requests:
cpu: 10m
memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
name: default-http-backend
namespace: kube-system
labels:
app.kubernetes.io/name: default-http-backend
spec:
ports:
- port: 80
targetPort: 8080
selector:
app.kubernetes.io/name: default-http-backend
EOF
kubectl apply -f deploy.yaml
kubectl apply -f backend.yaml
cat > ingress-demo-app.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello-server
spec:
replicas: 2
selector:
matchLabels:
app: hello-server
template:
metadata:
labels:
app: hello-server
spec:
containers:
- name: hello-server
image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/hello-server
ports:
- containerPort: 9000
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx-demo
name: nginx-demo
spec:
replicas: 2
selector:
matchLabels:
app: nginx-demo
template:
metadata:
labels:
app: nginx-demo
spec:
containers:
- image: nginx
name: nginx
---
apiVersion: v1
kind: Service
metadata:
labels:
app: nginx-demo
name: nginx-demo
spec:
selector:
app: nginx-demo
ports:
- port: 8000
protocol: TCP
targetPort: 80
---
apiVersion: v1
kind: Service
metadata:
labels:
app: hello-server
name: hello-server
spec:
selector:
app: hello-server
ports:
- port: 8000
protocol: TCP
targetPort: 9000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-host-bar
spec:
ingressClassName: nginx
rules:
- host: "hello.chenby.cn"
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: hello-server
port:
number: 8000
- host: "demo.chenby.cn"
http:
paths:
- pathType: Prefix
path: "/nginx"
backend:
service:
name: nginx-demo
port:
number: 8000
EOF
# 等创建完成后在执行：
kubectl apply -f ingress-demo-app.yaml
kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress-host-bar nginx hello.chenby.cn,demo.chenby.cn 192.168.206.32 80 7s
13.2 过滤查看ingress端口
# 修改为nodeport
kubectl edit svc -n ingress-nginx ingress-nginx-controller
type: NodePort
[root@hello ~/yaml]# kubectl get svc -A | grep ingress
ingress-nginx ingress-nginx-controller NodePort 10.104.231.36 <none> 80:32636/TCP,443:30579/TCP 104s
ingress-nginx ingress-nginx-controller-admission ClusterIP 10.101.85.88 <none> 443/TCP 105s
[root@hello ~/yaml]#
14. 附录
docker pull docker.io/aloisreitbauer/hello-server:v0.1.2
docker pull docker.io/calico/cni:master
docker pull docker.io/calico/node:master
docker pull docker.io/cilium/startup-script:2de58e53a7060593c91d0655b337a57f06bf5d66
docker pull docker.io/cnsre/ingress-nginx-kube-webhook-certgen:v1.0
docker pull docker.io/kubernetesui/dashboard:v2.7.0
docker pull docker.io/kubernetesui/metrics-scraper:v1.0.8
docker pull docker.io/library/busybox:1.28
docker pull docker.io/library/mysql:5.7
docker pull docker.io/willdockerhub/ingress-nginx-controller:v1.1.0
docker pull m.daocloud.io/registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.5
docker pull quay.io/cilium/certgen:v0.1.8
docker pull quay.io/cilium/cilium-etcd-operator:v2.0.7
docker pull quay.io/cilium/cilium:v1.12.6
docker pull quay.io/cilium/clustermesh-apiserver:v1.12.6
docker pull quay.io/cilium/hubble-relay:v1.12.6
docker pull quay.io/cilium/hubble-ui-backend:v0.9.2
docker pull quay.io/cilium/hubble-ui:v0.9.2
docker pull quay.io/cilium/operator:v1.12.6
docker pull quay.io/coreos/etcd:v3.5.4
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/cni:master
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/coredns:v1.10.0
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/kube-controllers:master
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/metrics-server:v0.5.2
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/node:master
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/pause:3.6
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/typha:master
docker pull docker.io/calico/kube-controllers:master
docker pull docker.io/calico/typha:master
docker pull docker.io/library/nginx:1.25.3
# docker 保存镜像
docker save docker.io/aloisreitbauer/hello-server:v0.1.2 -o hello-server.tar
docker save docker.io/calico/cni:master -o calico-cni.tar
docker save docker.io/calico/node:master -o calico-node.tar
docker save docker.io/cilium/startup-script:2de58e53a7060593c91d0655b337a57f06bf5d66 -o startup-script.tar
docker save docker.io/cnsre/ingress-nginx-kube-webhook-certgen:v1.0 -o kube-webhook-certgen.tar
docker save docker.io/kubernetesui/dashboard:v2.7.0 -o dashboard.tar
docker save docker.io/kubernetesui/metrics-scraper:v1.0.8 -o metrics-scraper.tar
docker save docker.io/library/busybox:1.28 -o busybox.tar
docker save docker.io/library/mysql:5.7 -o mysql.tar
docker save docker.io/willdockerhub/ingress-nginx-controller:v1.1.0 -o ingress-controller.tar
docker save m.daocloud.io/registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.5 -o cluster-proportional-autoscaler.tar
docker save quay.io/cilium/certgen:v0.1.8 -o certgen.tar
docker save quay.io/cilium/cilium-etcd-operator:v2.0.7 -o cilium-etcd-operator.tar
docker save quay.io/cilium/cilium:v1.12.6 -o cilium.tar
docker save quay.io/cilium/clustermesh-apiserver:v1.12.6 -o clustermesh-apiserver.tar
docker save quay.io/cilium/hubble-relay:v1.12.6 -o hubble-relay.tar
docker save quay.io/cilium/hubble-ui-backend:v0.9.2 -o hubble-ui-backend.tar
docker save quay.io/cilium/hubble-ui:v0.9.2 -o hubble-ui.tar
docker save quay.io/cilium/operator:v1.12.6 -o operator.tar
docker save quay.io/coreos/etcd:v3.5.4 -o etcd.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/cni:master -o cni.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/coredns:v1.10.0 -o coredns.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5 -o defaultbackend-amd64.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/kube-controllers:master -o kube-controllers.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/metrics-server:v0.5.2 -o metrics-server.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/node:master -o node.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/pause:3.6 -o pause.tar
docker save registry.cn-hangzhou.aliyuncs.com/chenby/typha:master -o typha.tar
docker save docker.io/calico/kube-controllers:master -o calico-kube-controllers.tar
docker save docker.io/calico/typha:master -o calico-typha.tar
docker save docker.io/library/nginx:1.25.3 -o nginx.tar
# 传输到各个节点
for NODE in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02; do scp -r images/ $NODE:/root/ ; done
# 创建命名空间
ctr ns create k8s.io
# 导入镜像
ctr ns create k8s.io
ctr --namespace k8s.io image import images/cni.tar
ctr --namespace k8s.io image import images/node.tar
ctr --namespace k8s.io image import images/typha.tar
ctr --namespace k8s.io image import images/kube-controllers.tar
ctr --namespace k8s.io image import images/coredns.tar
ctr --namespace k8s.io image import images/pause.tar
ctr --namespace k8s.io image import images/metrics-server.tar
ctr --namespace k8s.io image import images/dashboard.tar
ctr --namespace k8s.io image import images/metrics-scraper.tar
ctr --namespace k8s.io image import images/dashboard.tar
ctr --namespace k8s.io image import images/metrics-scraper.tar
ctr --namespace k8s.io image import images/cilium.tar
ctr --namespace k8s.io image import images/certgen.tar
ctr --namespace k8s.io image import images/hubble-relay.tar
ctr --namespace k8s.io image import images/hubble-ui-backend.tar
ctr --namespace k8s.io image import images/hubble-ui.tar
ctr --namespace k8s.io image import images/cilium-etcd-operator.tar
ctr --namespace k8s.io image import images/operator.tar
ctr --namespace k8s.io image import images/clustermesh-apiserver.tar
ctr --namespace k8s.io image import images/etcd.tar
ctr --namespace k8s.io image import images/startup-script.tar
ctr --namespace k8s.io image import images/kube-webhook-certgen.tar
ctr --namespace k8s.io image import images/ingress-controller.tar
ctr --namespace k8s.io image import images/cilium-etcd-operato.tar
ctr --namespace k8s.io image import images/cluster-proportional-autoscaler.tar
ctr --namespace k8s.io image import images/defaultbackend-amd64.tar
ctr --namespace k8s.io image import images/calico-cni.tar
ctr --namespace k8s.io image import images/calico-node.tar
ctr --namespace k8s.io image import images/busybox.tar
ctr --namespace k8s.io image import images/hello-server.tar
ctr --namespace k8s.io image import images/mysql.tar
ctr --namespace k8s.io image import images/calico-kube-controllers.tar
ctr --namespace k8s.io image import images/calico-typha.tar
ctr --namespace k8s.io image import images/nginx.tar
# 安装包只下载不安装
sudo yum -y install sshpass --downloadonly --downloaddir=./sshpass
sudo yum -y openssh = 7.4p1-21.el7 --downloadonly --downloaddir=./openssh
yum install -y sshpass
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
15. 问题：
1、解决The connection to the server localhost:8080 was refused - did you specify the right host or port?
kubectl命令需要使用kubernetes-admin来运行，需要admin.conf文件（conf文件是通过“ kubeadmin init”命令在主节点/etc/kubernetes 中创建），但是从节点没有conf文件，也没有设置 KUBECONFIG =/root/admin.conf环境变量，所以需要复制conf文件到从节点，并设置环境变量就OK了
2、Linux启动某个服务报错：Failed to start xxx.service: Unit not found
systemctl list-unit-files --type=service
systemctl daemon-reload
systemctl status xxx.service
journalctl -xefu kubelet
2、Failed to get the info of the filesystem with mountpoint" err="unable to find data in memory cache" mountpoint="/var/lib/containerd/io.containerd.sna
k8s二进制集群部署1.25.4
于 2023-12-05 00:22:23 首次发布
1997
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
