1. 安装篇-K8S安装-kubeadm

王茗渠

已于 2023-08-25 14:06:44 修改

阅读量229

点赞数 1

文章标签： kubernetes

于 2023-07-14 13:37:11 首次发布

本文链接：https://blog.csdn.net/weixin_40901913/article/details/131721425

版权

错误手册

0. Kubernetes-Error

集群规划

主机规划

服务器名称	主机名	主机IP地址	资源配置	节点用途
/	k8s-master-lb	192.169.1.200	/	keepalived虚拟IP
K8S-1	k8s-master-201	192.169.1.201	2核\|2G	master节点
K8S-2	k8s-master-202	192.169.1.202	2核\|2G	master节点
K8S-3	k8s-master-203	192.169.1.203	2核\|2G	master节点
K8S-4	k8s-worker-204	192.169.1.204	2核\|2G	worker节点
K8S-5	k8s-worker-205	192.169.1.205	2核\|2G	worker节点

软件版本

软件	版本	备注
kubernetes	v1.22.6
docker-ce	20.10.12
etcd	3.5.0-0
coredns	v1.8.4
calico	v3.21.4
harbor	v2.3.2
dashboard	v2.4.1
helm	3.8.0
metallb	v0.11.0
ingress-nginx	v1.1.1
metrics-server	v0.6.0
prometheus	v2.32.1
grafana	8.3.3
istio	1.11.1

基础环境配置

配置hostname

涉及节点：所有节点逐个操作

hostnamectl set-hostname k8s-master-201 && hostname
hostnamectl set-hostname k8s-master-202 && hostname
hostnamectl set-hostname k8s-master-203 && hostname
hostnamectl set-hostname k8s-worker-204 && hostname
hostnamectl set-hostname k8s-worker-205 && hostname

配置hosts

涉及节点：所有节点

cat >> /etc/hosts << 'EOF'
192.169.1.200 k8s-master-lb
192.169.1.201 k8s-master-201
192.169.1.202 k8s-master-202
192.169.1.203 k8s-master-203
192.169.1.204 k8s-worker-204
192.169.1.205 k8s-worker-205
EOF

ping k8s-master-201 -c 1
ping k8s-master-202 -c 1
ping k8s-master-203 -c 1
ping k8s-worker-204 -c 1
ping k8s-worker-205 -c 1

关闭firewalld

涉及节点：所有节点

systemctl disable --now firewalld
systemctl status firewalld | grep Active

关闭selinux

涉及节点：所有节点

setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

grep 'SELINUX=disabled' /etc/selinux/config

关闭NetworkManager

涉及节点：所有节点

systemctl stop NetworkManager
systemctl disable NetworkManager
systemctl status NetworkManager | grep Active

关闭dnsmasq

涉及节点：所有节点

systemctl stop dnsmasq
systemctl disable dnsmasq
systemctl status dnsmasq| grep Active

关闭swap

涉及节点：所有节点

swapoff -a && sysctl -w vm.swappiness=0
sed -ri 's/.*swap.*/#&/' /etc/fstab
grep 'swap' /etc/fstab

设置时区

涉及节点：所有节点

timedatectl set-timezone Asia/Shanghai

安装ntpdate

涉及节点：所有节点

安装ntpdate

yum -y install epel-release
yum -y install ntpdate

配置时间同步

手动同步时间

ntpdate time2.aliyun.com

定时同步时间

crontab -e
*/5 * * * * ntpdate time2.aliyun.com

开机同步时间

cat >> /etc/rc.d/rc.local << 'EOF'
ntpdate time2.aliyun.com
EOF
chmod +x /etc/rc.d/rc.local

免密登录

涉及节点：k8s-master-201
配置k8s-master-201节点可以免密登录其他节点，用于安装过程中生成的配置文件和证书均在k8s-master-201上操作，集群管理也在k8s-master-201上操作。

生成秘钥

ssh-keygen -t rsa

发送密钥到其他节点

yum -y install sshpass
for HOST_NAME in k8s-master-201 k8s-master-202 k8s-master-203 k8s-worker-204 k8s-worker-205
do
  sshpass -p "cmk521" ssh-copy-id -o "StrictHostKeyChecking no" -i .ssh/id_rsa.pub $HOST_NAME
done

配置源

涉及节点：所有节点

安装epel源

yum -y install epel-release

修改国内源

sed -e 's!^metalink=!#metalink=!g' \
    -e 's!^#baseurl=!baseurl=!g' \
    -e 's!//download\.fedoraproject\.org/pub!//mirrors.tuna.tsinghua.edu.cn!g' \
    -e 's!http://mirrors!https://mirrors!g' \
    -i /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel-testing.repo

安装常用软件

yum -y install net-tools \
    nmap-ncat \
    sysstat \
    git \
    ipset \
    ipvsadm \
    bash-completion \
    wget \
    unzip \
    lrzsz \
    lsof \
    vim

配置docker源

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo \
   -O /etc/yum.repos.d/docker-ce.repo

配置kubernetes源

cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name = kubernetes
baseurl = https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled = 1
gpgcheck =1
gpgkey = https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg \
  https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
EOF

内核配置

涉及节点：所有节点

CentOS7

centos7内核需要升级至4.18以上
升级内核

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm

yum --enablerepo=elrepo-kernel install kernel-ml-devel kernel-ml -y
yum --enablerepo=elrepo-kernel install kernel-lt-devel kernel-lt -y

修改默认启动内容

awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
grub2-set-default 0
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot

检查内核版本

uname -r

注意：
通过命令awk -F’ ‘$1=="menuentry " {print i++ " : " $2}’ /etc/grub2.cfg可以查看到可用内核，及内核的序号。
通过命令grub2-set-default 0，设置新内核为默认启动的内核。
通过命令grub2-mkconfig -o /boot/grub2/grub.cfg生成grub文件。
通过reboot启动服务器即可。

CentOS8

centos8按需升级
升级内核

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm

yum --enablerepo=elrepo-kernel install kernel-ml -y
yum --enablerepo=elrepo-kernel install kernel-lt -y

修改默认启动内容

#查看当前默认启动内核
grubby --default-kernel

#以新版本内核启动
grub2-set-default 0

#指定某个内核启动
grubby --set-default /boot/vmlinuz-5.19.2-1.el8.elrepo.x86_64

#重启主机
reboot

检查内核版本

uname -r

参数优化

涉及节点：所有节点

系统调优参数

cat > /etc/sysctl.d/k8s.conf <<EOF
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
# 要求iptables不对bridge的数据进行处理
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
vm.overcommit_memory=1
# 开启OOM
vm.panic_on_oom=0
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.swappiness=0
# ipvs优化
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
EOF

sysctl --system

文件最大打开数

cat > /etc/security/limits.d/k8s.conf <<EOF
*       soft    nproc   1048576
*       hard    nproc   1048576
*       soft    nofile  1048576
*       hard    nofile  1048576
root    soft    nproc   1048576
root    hard    nproc   1048576
root    soft    nofile  1048576
root    hard    nofile  1048576
EOF

优化日志处理，减少磁盘IO

sed -ri 's/^\$ModLoad imjournal/#&/' /etc/rsyslog.conf
sed -ri 's/^\$IMJournalStateFile/#&/' /etc/rsyslog.conf

sed -ri 's/^#(DefaultLimitCORE)=/\1=100000/' /etc/systemd/system.conf
sed -ri 's/^#(DefaultLimitNOFILE)=/\1=100000/' /etc/systemd/system.conf

ssh 连接优化

sed -ri 's/^#(UseDNS )yes/\1no/' /etc/ssh/sshd_config

基本组件安装

安装ipvs模块

涉及节点：所有节点

cat > /etc/modules-load.d/k8s-ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

for mod in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack; do
    modprobe $mod
done

安装docker

涉及节点：所有节点
注意
docker版本选择参考https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/，如果对应k8s版本没有docker版本信息，则去上一个版本查找。

docker依赖

yum install -y yum-utils
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

docker可用版本（可忽略执行）

yum list docker-ce --showduplicates
yum list containerd.io --showduplicate
yum list docker-ce-rootless-extras --showduplicate

docker安装

yum -y install docker-ce-20.10.24-3.el7 \
docker-ce-cli-20.10.24-3.el7 \
docker-ce-rootless-extras-20.10.24-3.el7 \
containerd.io

docker配置

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": [
    "https://mciwm180.mirror.aliyuncs.com",
    "https://docker.mirrors.ustc.edu.cn/",
    "https://registry.docker-cn.com"
  ],
  "log-driver": "json-file",
  "log-opts": {
    "max-file": "10",
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

docker启动

systemctl enable --now docker
systemctl enable --now containerd

docker当前版本

docker version

安装k8s组件

涉及节点：所有节点
k8s组件可用版本（可忽略执行）

yum list kubeadm.x86_64 --showduplicates | sort -r

k8s组件安装

yum -y install kubeadm-1.22.6 \
kubectl-1.22.6 \
kubelet-1.22.6

k8s组件当前版本

kubeadm version
kubectl version
kubelet --version

启动kubelet

systemctl daemon-reload
systemctl enable --now kubelet

注意
kubelet可能启动失败，原因是没有设置集群初始化。

命令补全

yum -y install bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

高可用组件安装

haproxy安装

涉及节点：所有master节点
安装haproxy

yum -y install haproxy

配置haproxy
所有master节点配置相同

cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg_`date +%F`
HOST_NAME_1=k8s-master-201
HOST_IP_1=192.169.1.201
HOST_NAME_2=k8s-master-202
HOST_IP_2=192.169.1.202
HOST_NAME_3=k8s-master-203
HOST_IP_3=192.169.1.203
cat > /etc/haproxy/haproxy.cfg << EOF
global
  maxconn 2000
  ulimit-n 16384
  log 127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode http
  option httplog
  timeout connect 5000
  timeout client 50000
  timeout server 50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend monitor-in
  bind *:33305
  mode http
  option httplog
  monitor-uri /monitor

listen stats
  bind *:8006
  mode http
  stats enable
  stats hide-version
  stats uri /stats
  stats refresh 30s
  stats realm Haproxy\ Statistics
  stats auth admin:admin

frontend k8s-master
  bind 0.0.0.0:16443
  bind 127.0.0.1:16443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server $HOST_NAME_1 $HOST_IP_1:6443 check
  server $HOST_NAME_2 $HOST_IP_2:6443 check
  server $HOST_NAME_3 $HOST_IP_3:6443 check
EOF

启动haproxy

systemctl enable --now haproxy

keepalived安装

涉及节点：所有master节点
安装keepalived

yum -y install keepalived

配置keepalived（k8s-master-201）

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_`date +%F`
mkdir -p /etc/keepalived/script


INTERFACE=ens32
HOST_IP=192.169.1.201
HOST_VIP=192.169.1.200
TYPES=MASTER
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
  router_id LVS_DEVEL
}

vrrp_script chk_apiserver {
  script "/etc/keepalived/script/check_apiserver.sh"
  interval 2
  weight -5
  fall 3
  rise 2
}

vrrp_instance VI_1 {
  state MASTER
  interface $INTERFACE
  mcast_src_ip $HOST_IP
  virtual_router_id 51
  priority 100
  advert_int 2
  authentication {
    auth_type PASS
    auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
    $HOST_VIP
  }
#  track_script {
#    chk_apiserver
#  }
}
EOF

配置keepalived（k8s-master-202）

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_`date +%F`
mkdir -p /etc/keepalived/script


INTERFACE=ens32
HOST_IP=192.169.1.202
HOST_VIP=192.169.1.200
TYPES=BACKUP
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
  router_id LVS_DEVEL
}

vrrp_script chk_apiserver {
  script "/etc/keepalived/script/check_apiserver.sh"
  interval 2
  weight -5
  fall 3
  rise 2
}

vrrp_instance VI_1 {
  state $TYPES
  interface $INTERFACE
  mcast_src_ip $HOST_IP
  virtual_router_id 51
  priority 100
  advert_int 2
  authentication {
    auth_type PASS
    auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
    $HOST_VIP
  }
#  track_script {
#    chk_apiserver
#  }
}
EOF

配置keepalived（k8s-master-203）

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_`date +%F`
mkdir -p /etc/keepalived/script


INTERFACE=ens32
HOST_IP=192.169.1.203
HOST_VIP=192.169.1.200
TYPES=BACKUP
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
  router_id LVS_DEVEL
}

vrrp_script chk_apiserver {
  script "/etc/keepalived/script/check_apiserver.sh"
  interval 2
  weight -5
  fall 3
  rise 2
}

vrrp_instance VI_1 {
  state $TYPES
  interface $INTERFACE
  mcast_src_ip $HOST_IP
  virtual_router_id 51
  priority 100
  advert_int 2
  authentication {
    auth_type PASS
    auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
    $HOST_VIP
  }
#  track_script {
#    chk_apiserver
#  }
}
EOF

配置健康检查（所有master节点）

cat > /etc/keepalived/script/check_apiserver.sh << "EOF"
!#/bin/bash

ERR=0
for k in $(seq 1 5)
do
  	CHECK_CODE=$(pgrep kube-apiserver)
  	if [[ $CHECK_CODE = "" ]]; then
      	ERR=$(expr $ERR + 1)
      	sleep 5
      	continue
  	else
      	ERR=0
      	break
  	fi
done

if [[ $ERR != "0" ]]; then
  	echo "systemctl stop keepalived"
  	/usr/bin/systemctl stop keepalived
  	exit 1
else
  	exit 0
fi
EOF

启动keepalived

systemctl enable --now keepalived

注意
keepalived健康监控已经注释，等待集群部署完成后再取消注释重启keepalived。

集群初始化

配置kubeadm初始化文件

涉及节点：所有master节点
生成kubeadm文件

kubeadm config print init-defaults --component-configs \
KubeProxyConfiguration,KubeletConfiguration > kubeadm-config.yaml

# 查看所需镜像
kubeadm config images list --config kubeadm-config.yaml

编辑kubeadm文件

HOST_IP="192.169.1.201"
HOST_NAME="k8s-master-201"
REGISTRY="registry.cn-hangzhou.aliyuncs.com/google_containers"
VERSION="1.22.0"

cat > ./kubeadm-config.yaml << EOF
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  #advertiseAddress: 1.2.3.4
  advertiseAddress: $HOST_IP
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  imagePullPolicy: IfNotPresent
  #name: node
  name: $HOST_NAME
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
#imageRepository: k8s.gcr.io
imageRepository: $REGISTRY
kind: ClusterConfiguration
#kubernetesVersion: 1.22.0
kubernetesVersion: $VERSION
controlPlaneEndpoint: $HOST_IP:6443
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
  maxPerCore: null
  min: null
  tcpCloseWaitTimeout: null
  tcpEstablishedTimeout: null
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: false
  syncPeriod: 0s
  tcpFinTimeout: 0s
  tcpTimeout: 0s
  udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: ""
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:
  enableDSR: false
  networkName: ""
  sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
EOF

kubeadm-config.yaml
检查文件

kubeadm init --config ./kubeadm-config.yaml --dry-run

下载镜像

kubeadm config images pull --config ./kubeadm-config.yaml

初始化k8s集群

涉及节点：所有节点
初始化k8s-master-201节点后，会在/etc/kubernetes目录下生成对应的证书和配置。最后将其他master节点、node节点加入到k8s-master-201即可。
k8s-master-201

#文件初始化命令
kubeadm init --config ./kubeadm-config.yaml --upload-certs

初始化成功以后，会产生token值，用于其他节点加入使用

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:
# 配置管理节点
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:
# 添加master节点
  kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4 \
	--control-plane --certificate-key 4fcb65ba01180c346087ca289fbddf445af82e692bac65636b63f2ddf7d06b6c

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:
# 添加node节点
kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4

如果初始化失败，需要重置后再初始化

#初始化重置命令
kubeadm reset

使用命令也可以初始化（与配置文件初始化二选一）

kubeadm init \
--apiserver-advertise-address=192.169.1.201  \
--image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version v1.27.0 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--ignore-preflight-errors=all

k8s集群添加节点

如果key过期可以使用如下命令生成新的key。
添加node节点
在k8s-master-201上生成node的token

kubeadm token create --print-join-command

# token结果
kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4

再k8s-worker-204执行命令添加k8s-worker-204为node节点

kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4

再k8s-worker-205执行命令添加k8s-worker-205为node节点

kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4

添加master节点
在k8s-master-201上生成master的token

kubeadm token create --print-join-command
# token结果
kubeadm join 192.169.1.201:6443 --token b84592.0w9ies5r5nj44tr7 --discovery-token-ca-cert-hash sha256:7b6738e46023e9bd7149e62977fc64ed40cc45f437f3a15ac790af5bb14cc7ca 

kubeadm init phase upload-certs --upload-certs
# token结果
I0516 17:19:58.320947   12440 version.go:255] remote version is much newer: v1.27.1; falling back to: stable-1.22
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
f4559139e998796ba23246d6c0a3d02b262bffeb405c4ee21d9370712662cf52

再k8s-master-202执行命令添加k8s-master-202为master节点

kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4 \
--control-plane --certificate-key 4fcb65ba01180c346087ca289fbddf445af82e692bac65636b63f2ddf7d06b6c

再k8s-master-203执行命令添加k8s-master-203为master节点

kubeadm join 192.169.1.201:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:f38f229ac8c52df013958675b5f34dc087cbb05fceabf2812cc79507019b10b4 \
--control-plane --certificate-key 4fcb65ba01180c346087ca289fbddf445af82e692bac65636b63f2ddf7d06b6c

k8s集群管理节点

k8s-master-201
执行如下命令可以将k8s-master-201当做管理节点

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

检查集群节点

kubectl get nodes

NAME             STATUS     ROLES                  AGE     VERSION
k8s-master-201   NotReady   control-plane,master   4m24s   v1.22.6
k8s-master-202   NotReady   control-plane,master   3m43s   v1.22.6
k8s-master-203   NotReady   control-plane,master   3m39s   v1.22.6
k8s-worker-204   NotReady   <none>                 94s     v1.22.6
k8s-worker-205   NotReady   <none>                 110s    v1.22.6

注意
各个节点NotReady的原因是没有安装网络插件。网络插件有：flannel、calico、canal、kube-router、weave net等，这里使用calico网络插件。

k8s集群环境变量

涉及节点：所有master节点

cat >> /root/.bashrc << EOF
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source /root/.bashrc

核心组件安装

安装calico

涉及节点：管理节点k8s-master-201节点
下载calico

curl https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/calico.yaml -O

calico.yaml
编辑calico

vim ./calico.yaml

安装calico

kubectl apply -f ./calico.yaml

安装metrics-server

涉及节点：管理节点k8s-master-201节点
下载metrics-server

wget -O /root/metrics-server-v0.6.0.yaml \
https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.0/components.yaml

metrics-server-v0.6.0.yaml
修改metrics-server配置

sed -i 's@k8s.gcr.io/metrics-server@k8simage@' /root/metrics-server-v0.6.0.yaml
sed -i '/args:/a\        - --kubelet-insecure-tls' /root/metrics-server-v0.6.0.yaml

安装metrics-server

kubectl apply -f /root/metrics-server-v0.6.0.yaml

验证metrics-server
需要等待 metrics-server 的 pod 就绪之后再等它收集一会再看。
查看node情况

kubectl top nodes

NAME             CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
k8s-master-201   168m         8%     1362Mi          73%       
k8s-master-202   121m         6%     1228Mi          66%       
k8s-master-203   122m         6%     1211Mi          65%       
k8s-worker-204   49m          2%     850Mi           46%       
k8s-worker-205   46m          2%     823Mi           44%

查看pod情况

kubectl top pods -n kube-system

NAME                                       CPU(cores)   MEMORY(bytes)   
calico-kube-controllers-5b67bc688c-g44xc   1m           24Mi            
calico-node-89t7x                          15m          117Mi           
calico-node-bghvw                          20m          91Mi            
calico-node-cdszv                          15m          120Mi           
calico-node-d4czf                          17m          91Mi            
calico-node-h2rx6                          16m          118Mi           
coredns-7d89d9b6b8-c485g                   1m           21Mi            
coredns-7d89d9b6b8-mxvmh                   1m           16Mi            
etcd-k8s-master-201                        23m          83Mi            
etcd-k8s-master-202                        20m          85Mi            
etcd-k8s-master-203                        20m          83Mi            
kube-apiserver-k8s-master-201              47m          398Mi           
kube-apiserver-k8s-master-202              43m          321Mi           
kube-apiserver-k8s-master-203              40m          301Mi           
kube-controller-manager-k8s-master-201     13m          81Mi            
kube-controller-manager-k8s-master-202     1m           45Mi            
kube-controller-manager-k8s-master-203     1m           41Mi            
kube-proxy-4q47s                           1m           30Mi            
kube-proxy-bbn4c                           1m           17Mi            
kube-proxy-cpqzr                           1m           19Mi            
kube-proxy-wv4qh                           1m           30Mi            
kube-proxy-z8xfk                           1m           16Mi            
kube-scheduler-k8s-master-201              2m           36Mi            
kube-scheduler-k8s-master-202              2m           38Mi            
kube-scheduler-k8s-master-203              2m           31Mi            
metrics-server-f657d6869-mmfns             2m           22Mi

安装dashboard

涉及节点：管理节点k8s-master-201节点
下载dashboard

wget -O dashboard-v2.4.0.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v2.4.0/aio/deploy/recommended.yaml

dashboard-v2.4.0.yaml
修改dashboard配置

# 使用 NodePort
sed -i '39a\  type: NodePort' dashboard-v2.4.0.yaml
sed -i '43a\      nodePort: 30443' dashboard-v2.4.0.yaml

# 使用 LoadBalancer
sed -i '39a\  type: LoadBalancer' dashboard-v2.4.0.yaml

# 创建自定义证书时需要
sed -i '48,58d' dashboard-v2.4.0.yaml 

# 镜像拉取策略
sed -i '/imagePullPolicy/s@Always@IfNotPresent@' dashboard-v2.4.0.yaml

安装dashboard

kubectl create -f dashboard-v2.4.0.yaml

创建kubernetes-dashboard-certs

openssl genrsa -out dashboard.key 2048
openssl req -days 36500 -new -out dashboard.csr -key dashboard.key -subj '/CN=dashboard-cert'
openssl x509 -req -in dashboard.csr -signkey dashboard.key -out dashboard.crt

kubectl create secret generic kubernetes-dashboard-certs --from-file=dashboard.key \
  --from-file=dashboard.crt -n kubernetes-dashboard

安装 dashboard 的 serviceAccount

cat > ./dashboard-admin.yaml<<EOF
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: dashboard-admin
  namespace: kubernetes-dashboard

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: dashboard-admin-bind-cluster-role
  labels:
    k8s-app: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: dashboard-admin
  namespace: kubernetes-dashboard
EOF

kubectl create -f ./dashboard-admin.yaml

查看dashboard

kubectl get pod -n kubernetes-dashboard
NAME                                        READY   STATUS              RESTARTS   AGE
dashboard-metrics-scraper-c45b7869d-84qvb   1/1     Running             0          4m46s
kubernetes-dashboard-74979b6cf6-4vcxt       1/1     Running             0          4m46s

查看dashboard端口

kubectl get svc -A |grep kubernetes-dashboard
kubernetes-dashboard   dashboard-metrics-scraper   ClusterIP   10.99.19.162    <none>        8000/TCP                 4m2s
kubernetes-dashboard   kubernetes-dashboard        NodePort    10.99.170.130   <none>        443:30443/TCP            4m2s

登录dashboard
查看token值（token值即dashboard的登录密码）

kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret \
  | grep dashboard-admin | awk '{print $1}') | grep token: | awk '{print $2}'

eyJhbGciOiJSUzI1NiIsImtpZCI6InF0dEdjcFpGRjBVRllkcnM4TEVPVnMxV0Z0QkJwc0lKMlUyWE5RYlRaUnMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkZWZhdWx0LXRva2VuLXFoMjRwIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImRlZmF1bHQiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJlMzFmZGJmMS1mMDgyLTQ5YjQtYTJiNi1hZDQ3NDFmYTk4Y2EiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6ZGVmYXVsdCJ9.Rg-hW9eZcjZd6qGUHUA_mQqaRikE2OMN8_Z6gvLwl7TeKD7jh4R0wQjFpJ1qWGco-KeWrbeNvIE6QYVlS0vGD6SUhwe4ADeEhU2bEHfGjSCMjlm91PMGb2Gc5yKuN4nU_jP6OTWlNFKar3OFUDU4LK2IxpzXQ05KttmzTBwZfYK3vsyaQsMuQ0YaRQ7_0ClUJhf9MSpi3otvBFm4U4gf_eH2S7_HpFml2bCGzDET3kiymIL-ghjzykWgpBvb1_E4pvEXNHEnanGhzDNdEO35tqWSe_I_QE9K5OOcSxpQh7asMfgczr0HFsIhGZNzY98rErEMVgmZpDQS-eP1PHPBsw
eyJhbGciOiJSUzI1NiIsImtpZCI6InF0dEdjcFpGRjBVRllkcnM4TEVPVnMxV0Z0QkJwc0lKMlUyWE5RYlRaUnMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi12eG5jciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjVkODZmMWU1LTYwMzctNDczYy1hYjc0LTJjNDg1ZTZmYmRkMiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.twjKSQh4RX4jFW9zNaPP4KjF8VxD8MwdywFOrEDpiBlOjIH1Y1crL5Z6lr-g8TRFCNJ4N8X_cpfKtWiJxM3TfbEAfpMpgUoQWY7dVTSiSiCQk0ODN3hF4qlsnDxCuq44KEoD9wCByvRb6OhSS7u4BMLndxA3mzPNXZMlx2rzHSPPhusH2MxgMYdE-aVv4ZQfq7MbJLDePWUL1gGDWyHN1T4Yz8R9LkX5efNZ-0JOkzdmT7e6-5aHlCkC9uoADrgNFuUFR5iwYSOgRRnDDlZvknVFeZVlz9szofdKD2Gj7cSyfTcvx86BOkCIVmsoyw4FLaoL4PBEGh4LFAWwz8d5JQ

登录页面http://192.169.1.201:30443/

王茗渠

关注

1
点赞
踩
0

收藏

觉得还不错? 一键收藏
1
评论
1. 安装篇-K8S安装-kubeadm

配置k8s-master-201节点可以免密登录其他节点，用于安装过程中生成的配置文件和证书均在k8s-master-201上操作，集群管理也在k8s-master-201上操作。再k8s-master-202执行命令添加k8s-master-202为master节点。再k8s-master-203执行命令添加k8s-master-203为master节点。再k8s-worker-204执行命令添加k8s-worker-204为node节点。在k8s-master-201上生成master的token。
复制链接

扫一扫