k8s -- kubeadm高可用部署

keepalived参考地址：https://blog.csdn.net/mofiu/article/details/76644012

一、部署环境准备

• 所有节点都部署的准备工作

1. 设置主机名

master01 – 10.4.7.11
master02 – 10.4.7.12
master03 – 10.4.7.13
worker01 – 10.4.7.14
worker02 – 10.4.7.15

# 在10.4.7.11上执行
[root@localhost ~]# hostnamectl set-hostname master01 && bash

# 在10.4.7.12上执行
[root@localhost ~]# hostnamectl set-hostname master02 && bash

# 在10.4.7.13上执行
[root@localhost ~]# hostnamectl set-hostname master03 && bash

# 在10.4.7.14上执行
[root@localhost ~]# hostnamectl set-hostname worker01 && bash

# 在10.4.7.15上执行
[root@localhost ~]# hostnamectl set-hostname worker01 && bash

2. 关闭防火墙、swap，重置iptables

# 关闭防火墙
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld

# 重置iptables
[root@localhost ~]# iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat && iptables -P FORWARD ACCEPT

# 关闭swap
[root@localhost ~]# swapoff -a
[root@localhost ~]# sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab

# 关闭selinux
[root@localhost ~]# setenforce 0

# 关闭dnsmasq(否则可能导致docker容器无法解析域名)
[root@localhost ~]# service dnsmasq stop && systemctl disable dnsmasq

3. 安装依赖包

# 更新yum
[root@localhost ~]# yum update
# 安装依赖包
[root@localhost ~]# yum install -y conntrack ipvsadm ipset jq sysstat curl iptables libseccomp

4. 系统参数设置

# 制作配置文件
[root@localhost ~]# cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
EOF
# 生效文件
[root@localhost ~]# sysctl -p /etc/sysctl.d/kubernetes.conf

5. 安装docker

[root@localhost ~]# yum -y install yum-utils device-mapper-persistent-data lvm2 epel-release 
# 设置阿里镜像源
[root@localhost ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

[root@localhost ~]# yum install -y docker-ce

[root@localhost ~]# systemctl start docker

[root@localhost ~]# systemctl enable docker

[root@localhost ~]# systemctl status docker



[root@localhost ~]# cat > /etc/docker/daemon.json <<EOF
{
  "storage-driver": "overlay2",
  "insecure-registries": ["registry.access.redhat.com","quay.io","harbor.prod.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "live-restore": true
}
EOF
# "graph": "/data/docker", # 可以加上这个参数，是镜像、容器的存储路径，默认是/var/lib/docker

[root@localhost ~]# systemctl daemon-reload

[root@localhost ~]# systemctl restart docker

6. 各节点安装ipset服务，开启ip_vs

[root@localhost ~]# yum -y install ipvsadm ipset sysstat conntrack libseccomp

[root@localhost ~]# cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

[root@localhost ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules
[root@localhost ~]# sh /etc/sysconfig/modules/ipvs.modules 
[root@localhost ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4

二、安装kubeadm部署工具

1. 指定kubeadm工具下载yum源

[root@localhost ~]# cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

2. 下载安装相关工具

# 查看kubeadm版本
[root@localhost ~]# yum list kubeadm --showduplicates | sort -r

# 指定安装版本
[root@localhost ~]# yum install -y kubelet-1.19.9 kubeadm-1.19.9 kubectl-1.19.9

# 检查是否下载成功
[root@localhost ~]# rpm -qa | grep kube
kubernetes-cni-0.8.7-0.x86_64
kubectl-1.19.9-0.x86_64
kubelet-1.19.9-0.x86_64
kubeadm-1.19.9-0.x86_64

# 设置kubelet的cgroupdriver
# kubelet的cgroupdriver默认为systemd，如果上面没有设置docker的exec-opts为systemd，这里就需要将kubelet的设置为cgroupfs，如下：（这步可以省略）
# sed -i "s/cgroup-driver=systemd/cgroup-driver=cgroupfs/g" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

# 启动kubelet
[root@localhost ~]# systemctl enable kubelet && systemctl start kubelet

三、k8s高可用部署

1. master节点安装haproxy和keepalived服务

• 三台 master 节点上都执行（10.4.7.11 、10.4.7.12 、10.4.7.13）

1.1 安装 haproxy 和 keepalived

# 三个master节点都执行
[root@master01 ~]# yum -y install haproxy keepalived

1.2 修改三个master节点keepalived配置文件

master01节点上keepalived配置

# 备份原配置文件
[root@master01 ~]# cp -p /etc/keepalived/keepalived.conf /etc/keepalived/keepalived_example.conf

# 修改配置文件
[root@master01 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server localhost
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict                                   # 将这个参数注释掉，否则会造成vip无法ping通 
   vrrp_garp_interval 0
   vrrp_gna_interval 0
   # 在global_defs模块内添加如下两行内容
   script_user root                                # 设置运行脚本默认用户和组
   enable_script_security                          # 设置脚本只允许root用户运行
}

# 新增如下脚本检查模块check_haproxy
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"       # 定义检测脚本路径
    interval 3                                      # 指定脚本执行的间隔。单位是秒。默认为1s。
    weight -2                                       # 调整优先级<-254 ~ 254>。默认为2
    fall 10                                         # 执行失败多少次才认为失败。
    rise 2                                          # 执行成功多少次才认为是成功。
}

vrrp_instance VI_1 {
    state MASTER                                    # master01节点定义角色为MASTER
    interface ens33                                 # 本地网卡名称
    virtual_router_id 51
    priority 100                                    # 配置权重100，vip会优先调到权重高的节点上
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
    # 在这里定义vip地址
        10.4.7.253                                  # 定义了Vip地址
    }
    # track_script模块，循环执行脚本
    track_script {
        check_haproxy                               # 上面定义的脚本模块
    } 
}

master02节点上keepalived配置

# 备份原配置文件
[root@master02 ~]# cp -p /etc/keepalived/keepalived.conf /etc/keepalived/keepalived_example.conf

# 修改配置文件
[root@master02 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server localhost
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict                                   # 将这个参数注释掉，否则会造成vip无法ping通 
   vrrp_garp_interval 0
   vrrp_gna_interval 0
# 在global_defs模块内添加如下两行内容
   script_user root                                # 设置运行脚本默认用户和组
   enable_script_security                          # 设置脚本只允许root用户运行
}

# 新增如下脚本检查模块check_haproxy
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"       # 定义检测脚本路径
    interval 3                                      # 指定脚本执行的间隔。单位是秒。默认为1s。
    weight -2                                       # 调整优先级<-254 ~ 254>。默认为2
    fall 10                                         # 执行失败多少次才认为失败。
    rise 2                                          # 执行成功多少次才认为是成功。
}

vrrp_instance VI_1 {
    state BACKUP                                    # master02节点定义角色为BACKUP
    nopreempt                                       # 定义非抢占模式，以防master网络问题来回抢占
    interface ens33                                 # 本地网卡名称
    virtual_router_id 51
    priority 99                                     # 配置权重100，vip会优先调到权重高的节点上
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
# 在这里定义vip地址
        10.4.7.253                                  # 定义了Vip地址
    }
# track_script模块，循环执行脚本
    track_script {
        check_haproxy                               # 上面定义的脚本模块
    } 
}

master03节点上keepalived配置

# 备份原配置文件
[root@master03 ~]# cp -p /etc/keepalived/keepalived.conf /etc/keepalived/keepalived_example.conf

# 修改配置文件
[root@master03 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server localhost
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict                                   # 将这个参数注释掉，否则会造成vip无法ping通 
   vrrp_garp_interval 0
   vrrp_gna_interval 0
# 在global_defs模块内添加如下两行内容
   script_user root                                # 设置运行脚本默认用户和组
   enable_script_security                          # 设置脚本只允许root用户运行
}

# 新增如下脚本检查模块check_haproxy
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"       # 定义检测脚本路径
    interval 3                                      # 指定脚本执行的间隔。单位是秒。默认为1s。
    weight -2                                       # 调整优先级<-254 ~ 254>。默认为2
    fall 10                                         # 执行失败多少次才认为失败。
    rise 2                                          # 执行成功多少次才认为是成功。
}

vrrp_instance VI_1 {
    state BACKUP                                    # master03节点定义角色为BACKUP
    nopreempt                                       # 定义非抢占模式，以防master网络问题来回抢占
    interface ens33                                 # 本地网卡名称
    virtual_router_id 51
    priority 98                                     # 配置权重100，vip会优先调到权重高的节点上
    advert_int 1 
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
# 在这里定义vip地址
        10.4.7.253                                  # 定义了Vip地址
    }
# track_script模块，循环执行脚本
    track_script {
        check_haproxy                               # 上面定义的脚本模块
    } 
}

1.3 修改三个master节点haproxy配置文件

• 关于haproxy 的配置，三个master节点都相同，因此只需要修改一个节点，然后copy即可

[root@master01 ~]# cp -p  /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy_example.cfg

# 全部文件内容如下，对照修改即可
[root@master01 ~]# vim /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
#frontend  main *:5000
#    acl url_static       path_beg       -i /static /images /javascript /stylesheets
#    acl url_static       path_end       -i .jpg .gif .png .css .js
#
#    use_backend static          if url_static
#    default_backend             app
frontend  kubernetes-apiserver
    mode                        tcp
    bind                        *:16443
    option                      tcplog  
    default_backend             kubernetes-apiserver

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
#backend static
#    balance     roundrobin
#    server      static 127.0.0.1:4331 check

listen stats
    bind            *:1080
    stats auth      admin:awesomePassword
    stats refresh   5s
    stats realm     HAProxy\ Statistics
    stats uri       /admin?stats
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    balance          roundrobin
    server  master01 10.4.7.11:6443 check
    server  master02 10.4.7.12:6443 check
    server  master03 10.4.7.13:6443 check

[root@master01 ~]# scp /etc/keepalived/keepalived.conf 10.4.7.12:/etc/keepalived/keepalived.conf
[root@master01 ~]# scp /etc/keepalived/keepalived.conf 10.4.7.13:/etc/keepalived/keepalived.conf

1.4 在三个master节点上写haproxy健康检查脚本

[root@master01 ~]# cat >/etc/keepalived/check_haproxy.sh <<'EOF'
#!/bin/sh
# HAPROXY down
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ]
then
    systmectl start haproxy
    if [ $(ps -C haproxy --no-header | wc -l) -eq 0 ]
    then
        killall -9 haproxy
        echo "HAPROXY down" | mail -s "haproxy"
        sleep 360
    fi 
fi
EOF

[root@master01 ~]# chmod +x /etc/keepalived/check_haproxy.sh

[root@master01 ~]# scp /etc/keepalived/check_haproxy.sh 10.4.7.12:/etc/keepalived/check_haproxy.sh
[root@master01 ~]# scp /etc/keepalived/check_haproxy.sh 10.4.7.13:/etc/keepalived/check_haproxy.sh

1.5 在三台master上启动keepalived和haproxy

[root@master01 ~]# systemctl start keepalived && systemctl enable keepalived
[root@master01 ~]# systemctl start haproxy && systemctl enable haproxy

2. 正式部署k8s

2.1 获取kubeadm默认配置文件

• 修改kubeadm默认配置文件，并 下载相关镜像

在master01上执行（只需要在一台master上执行就可以了）

[root@master01 ~]# kubeadm config print init-defaults > kubeadm-config.yaml

# 修改kubeadm默认配置文件
[root@master01 ~]# vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.4.7.11                # 本地ip地址
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: master.10.4.7.11                     # 节点名称
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "10.4.7.253:16443"     # 虚拟IP和haproxy端口
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers               # 镜像仓库地址，修改成阿里云的仓库地址（原本是国外源，会卡，如果有vpn，这个不用改）
kind: ClusterConfiguration
kubernetesVersion: v1.19.0
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"               # pod地址段
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

# 下载相关镜像
[root@master01 ~]# kubeadm config images pull --config kubeadm-config.yaml

# 可以通过这种方式修改
[root@master02 ~]# kubeadm config print init-defaults > kubeadm-config.yaml
[root@master02 ~]# reip=$(hostname -I |awk '{print $1}')
[root@master02 ~]# nodeName=$(str=$(hostname);echo ${str:0:6}-$(hostname -I |awk '{print $1}'))
[root@master02 ~]# havip='controlPlaneEndpoint: "10.4.7.253:16443"'        # 根据实际情况修改vip+haproxy的端口
[root@master02 ~]# podip='podSubnet: "10.244.0.0/16"'
[root@master02 ~]# alireg="registry.cn-hangzhou.aliyuncs.com/google_containers"
[root@master02 ~]# sed -i "/name: $(hostname)/s/name.*/name: ${nodeName}/" kubeadm-config.yaml
[root@master02 ~]# sed -i "/clusterName: /a${havip}" kubeadm-config.yaml
[root@master02 ~]# sed -i "/dnsDomain: /a$(echo "\ " ${podip})" kubeadm-config.yaml
[root@master02 ~]# sed -i "/advertiseAddress: /s/advertiseAddress.*/advertiseAddress: ${reip}/" kubeadm-config.yaml

[root@master03 ~]# cat >> kubeadm-config.yaml << EOF
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
EOF

# 下载相关镜像
[root@master02 ~]# kubeadm config images pull --config kubeadm-config.yaml

2.2 初始化集群节点

• 在master01上执行
• 执行产生的这些信息最好保留下来

[root@master01 ~]# kubeadm init --config kubeadm-config.yaml
W0608 14:48:17.402870   12982 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.19.0
[preflight] Running pre-flight checks
        [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.7. Latest validated version: 19.03
        [WARNING Hostname]: hostname "master01" could not be reached
        [WARNING Hostname]: hostname "master01": lookup master01 on 8.8.8.8:53: no such host
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local master01] and IPs [10.96.0.1 10.4.7.11 10.4.7.253]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [localhost master01] and IPs [10.4.7.11 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [localhost master01] and IPs [10.4.7.11 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[apiclient] All control plane components are healthy after 106.014604 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.19" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master01 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master01 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 10.4.7.253:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf \
    --control-plane

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.4.7.253:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf

# 非root用户执行下面三条命令
[root@master01 ~]# mkdir -p $HOME/.kube
[root@master01 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master01 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

• 若集群初始化失败重置集群

kubeadm reset

2.3 将证书及秘钥上传给其他master节点

• 在其它两个master节点创建以下目录存放秘钥和证书
• 然后上传证书和秘钥

[root@master01 ~]# ssh 10.4.7.12 "mkdir -p /etc/kubernetes/pki/etcd"
[root@master01 ~]# ssh 10.4.7.13 "mkdir -p /etc/kubernetes/pki/etcd"
# 上传给master02
[root@master01 ~]# scp /etc/kubernetes/pki/ca.* root@10.4.7.12:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/sa.* root@10.4.7.12:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/front-proxy-ca.* root@10.4.7.12:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/etcd/ca.* root@10.4.7.12:/etc/kubernetes/pki/etcd/
[root@master01 ~]# scp /etc/kubernetes/admin.conf root@10.4.7.12:/etc/kubernetes/
# 上传给master03
[root@master01 ~]# scp /etc/kubernetes/pki/ca.* root@10.4.7.13:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/sa.* root@10.4.7.13:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/front-proxy-ca.* root@10.4.7.13:/etc/kubernetes/pki/
[root@master01 ~]# scp /etc/kubernetes/pki/etcd/ca.* root@10.4.7.13:/etc/kubernetes/pki/etcd/
[root@master01 ~]# scp /etc/kubernetes/admin.conf root@10.4.7.13:/etc/kubernetes/

2.4 将master节点admin.conf证书复制到其他node节点

[root@master01 ~]# scp /etc/kubernetes/admin.conf root@10.4.7.14:/etc/kubernetes/
[root@master01 ~]# scp /etc/kubernetes/admin.conf root@10.4.7.15:/etc/kubernetes/

2.5 将另两个master加入控制节点

• 执行初始化集群时产生的命令
• 在另外两个master节点上执行

在master02上执行

[root@master02 ~]# kubeadm join 10.4.7.253:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf \
    --control-plane --node-name=worker-10.4.7.12
    
# 非root用户执行下面三条命令
[root@master02 ~]# mkdir -p $HOME/.kube
[root@master02 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master02 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

在master03上执行

[root@master03 ~]# kubeadm join 10.4.7.253:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf \
    --control-plane --node-name=worker-10.4.7.13
    
# 非root用户执行下面三条命令
[root@master03 ~]# mkdir -p $HOME/.kube
[root@master03 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master03 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

2.6 将两个node节点加入集群

• 在 worker01上执行 （加上"--node-name=节点名称"参数，可以指定节点名称）

[root@worker01 ~]# kubeadm join 10.4.7.253:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf --node-name=worker-10.4.7.14

• 在 worker02上执行

[root@worker02 ~]# kubeadm join 192.168.2.100:16443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:a17804d9aa1f198f6530269df2c10fed923570dc2acbde9706490184ee876acf --node-name=worker-10.4.7.15

2.7 安装网络插件

• 先查看当前节点状态，目前都是 NotReady 状态

[root@master01 ~]# kubectl get nodes
NAME               STATUS     ROLES    AGE     VERSION
master-10.4.7.11   NotReady   master   65m     v1.19.9
master-10.4.7.12   NotReady   master   19m     v1.19.9
master-10.4.7.13   NotReady   master   24m     v1.19.9
worker-10.4.7.14   NotReady   <none>   2m25s   v1.19.9
worker-10.4.7.15   NotReady   <none>   3m20s   v1.19.9

• 安装网络插件(在一个master节点执行即可)

[root@master01 ~]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

• 等待网络插件pod都启动完成，再次查看节点状态

[root@master01 ~]# kubectl get pod -n kube-system -owide|grep flannel
kube-flannel-ds-cqxks              1/1     Running   0          14m   10.4.7.15   worker02   <none>           <none>
kube-flannel-ds-fvkbp              1/1     Running   0          14m   10.4.7.14   worker01   <none>           <none>
kube-flannel-ds-pvcmd              1/1     Running   0          14m   10.4.7.12   master02   <none>           <none>
kube-flannel-ds-xcsk4              1/1     Running   0          12m   10.4.7.13   master03   <none>           <none>
kube-flannel-ds-xncbk              1/1     Running   0          14m   10.4.7.11    master01   <none>           <none>


[root@master01 ~]# kubectl get nodes
NAME       STATUS   ROLES    AGE   VERSION
master-10.4.7.11   Ready    master   72m   v1.19.9
master-10.4.7.12   Ready    master   27m   v1.19.9
master-10.4.7.13   Ready    master   32m   v1.19.9
worker-10.4.7.14   Ready    <none>   10m   v1.19.9
worker-10.4.7.15   Ready    <none>   11m   v1.19.9

2.8 、下载etcdctl客户端命令行工具

下载工具包

wget https://github.com/etcd-io/etcd/releases/download/v3.4.14/etcd-v3.4.14-linux-amd64.tar.gz

解压并加入环境变量

tar -zxf etcd-v3.4.14-linux-amd64.tar.gz

mv etcd-v3.4.14-linux-amd64/etcdctl /usr/local/bin

chmod +x /usr/local/bin/

查看etcd高可用集群健康状态

[root@master01 ~]# ETCDCTL_API=3 etcdctl \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key \
--write-out=table \
--endpoints=10.4.7.11:2379,10.4.7.12:2379,10.4.7.13:2379 \
endpoint health

+--------------------+--------+-------------+-------+
|      ENDPOINT      | HEALTH |    TOOK     | ERROR |
+--------------------+--------+-------------+-------+
| 10.4.7.11:2379     |   true | 19.782159ms |       |
| 10.4.7.12:2379     |   true | 20.014403ms |       |
| 10.4.7.13:2379     |   true | 28.543079ms |       |
+--------------------+--------+-------------+-------+

查看etcd高可用集群成员列表

[root@master01 ~]# ETCDCTL_API=3 etcdctl \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key \
--write-out=table \
--endpoints=10.4.7.11:2379,10.4.7.12:2379,10.4.7.13:2379 \
member list

+------------------+---------+----------+----------------------------+----------------------------+------------+
|        ID        | STATUS  |   NAME   |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+----------+----------------------------+----------------------------+------------+
| 1c422186d56f7d31 | started | master01 |  https://10.4.7.11:2380 |  https://10.4.7.11:2379 |      false |
| 1d722801456abd95 | started | master03 | https://10.4.7.13:2380 | https://10.4.7.13:2379 |      false |
| 2b88f5e384805b4a | started | master02 | https://10.4.7.12:2380 | https://10.4.7.12:2379 |      false |
+------------------+---------+----------+----------------------------+----------------------------+------------+

查看etcd高可用集群 leader

[root@master01 ~]# ETCDCTL_API=3 etcdctl \
--cacert=/etc/kubernetes/pki/etcd/ca.crt \
--cert=/etc/kubernetes/pki/etcd/peer.crt \
--key=/etc/kubernetes/pki/etcd/peer.key \
--write-out=table \
--endpoints=10.4.7.11:2379,10.4.7.13:2379,10.4.7.12:2379 \
endpoint status

+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|      ENDPOINT      |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 10.4.7.11:2379     | 1c422186d56f7d31 |  3.4.13 |  3.2 MB |      true |      false |         5 |      17663 |              17663 |        |
| 10.4.7.13:2379     | 1d722801456abd95 |  3.4.13 |  3.2 MB |     false |      false |         5 |      17663 |              17663 |        |
| 10.4.7.12:2379     | 2b88f5e384805b4a |  3.4.13 |  3.3 MB |     false |      false |         5 |      17663 |              17663 |        |
+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

2.9 查看各组件状态异常

• 如果发现以下 scheduler 和 controller-manager 不健康状态报错，可以将三台master上的 /etc/kubernetes/manifests 目录下的 kube-scheduler.yaml 和 kube-controller-manager.yaml 文件中 “- --port=0” 注释掉

报错现象：

[root@master01 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                       ERROR
scheduler            Unhealthy   Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager   Unhealthy   Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
etcd-0               Healthy     {"health":"true"}

解决过程：在三台master上执行下面两条命令

sed -i '/- --port=0/s/^/#/' /etc/kubernetes/manifests/kube-scheduler.yaml

sed -i '/- --port=0/s/^/#/' /etc/kubernetes/manifests/kube-controller-manager.yaml

• 再次查看组件状态

[root@master01 manifests]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-0               Healthy   {"health":"true"}

---

---

---

外挂etcd的kubeadm-config.yaml高可用参考模板

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 172.16.3.130
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: master
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
  certSANs:
  - "node0"
  - "node1"
  - "node2"
  - "172.16.3.110"
  - "172.16.3.130"
  - "172.16.3.131"
  - "172.16.3.132"
  - "127.0.0.1"
  extraArgs: 
           etcd-cafile: /etc/etcd/ssl/ca.pem
           etcd-certfile: /etc/etcd/ssl/etcd.pem
           etcd-keyfile: /etc/etcd/ssl/etcd-key.pem
controlPlaneEndpoint: "172.16.3.130:6443"
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
   external:
        caFile: /etc/etcd/ssl/ca.pem
        certFile: /etc/etcd/ssl/etcd.pem
        keyFile: /etc/etcd/ssl/etcd-key.pem
        endpoints:
        - https://172.16.3.130:2379
        - https://172.16.3.131:2379
        - https://172.16.3.132:2379
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.19.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: "10.244.0.0/16"
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs" 

参考地址：https://blog.csdn.net/xixihahalelehehe/article/details/108184267