k8s 二进制安装多master节点高可用集群

•  k8s 集群环境规划：系统统一使用 CentOS 7.9

  192.168.1.10	k8s-m1	控制节点1
  192.168.1.11	k8s-m2	控制节点2
  192.168.1.20	k8s-w1	node节点1
  192.168.1.21	k8s-w2	node节点2
  控制节点高可用	keepalived + nginx	VIP192.168.1.100
  node节点高可用	keepalived + nginx	VIP192.168.1.101
  Pod 网段	10.0.0.0/16
  Service 网段	10.255.0.0/16

• 准备工作

  配置静态 IP、主机名、hosts 文件、主机之间无密码登录手动设置

  然后关闭 firewalld 防火墙、SElinux、交换分区 swap、修改内核参数、配置阿里云 repo 源、配置时间同步、安装基础软件包、安装 docker-ce、配置 docker 镜像加速器等操作
• 使用 SHELL 脚本部署

  #!/bin/bash
  
  function KILLPROC(){
  	echo $1 | xargs kill -9 &> /dev/null
  }
  
  function PROC_NAME(){
      printf "%-45s" ${1}
      tput sc
      while true
      do
          for ROATE in '-' "\\" '|' '/'
          do
              tput rc && tput ed
              printf "\033[1;33m%-s\033[0m" ${ROATE}
              sleep 0.5
          done
      done
  }
  
  function CHECK_STATUS(){
      if [ $? == 0 ];then
          KILLPROC ${1} &> /dev/null
          tput rc  && tput ed
          printf "\033[1;32m%-7s\033[0m\n" 'SUCCESS'
      else
          KILLPROC ${1} &> /dev/null
          tput rc  && tput ed
          printf "\033[1;31m%-7s\033[0m\n" 'FAILED'
      fi
  }
  
  function CHECK_SYSTEM(){
      PROC_NAME CHECK_SYSTEM &
      PROC_PID=$!
  if [[ ! `egrep -o "Red Hat" /proc/version` ]]; then
      echo "Must be Red Hat !"
      exit 1;
  fi
      CHECK_STATUS ${PROC_PID}
  }
  
  function CHECK_USER(){
      PROC_NAME CHECK_USER &
      PROC_PID=$!
  if [[ `id -u` != "0" ]]; then
  	echo "Must be root !"
  	exit 2;
  fi
      CHECK_STATUS ${PROC_PID} 
  }
  
  function CHECK_NETWORK(){
      PROC_NAME CHECK_NETWORK &
      PROC_PID=$!
  if [[ ! `ping -c2 www.baidu.com` ]]; then
  	echo "NetWork is not ready !"
  	exit 3;
  fi
      CHECK_STATUS ${PROC_PID} 
  }
  
  function ADD_K8S_REPO(){
      PROC_NAME ADD_K8S_REPO &
      PROC_PID=$!
  
  cat <<EOF > /etc/yum.repos.d/kubernetes.repo
  [kubernetes]
  name=Kubernetes
  baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  enabled=1
  gpgcheck=0
  EOF
      CHECK_STATUS ${PROC_PID}
  }
  
  function YUM_BASE_INSTALL(){
      PROC_NAME YUM_BASE_INSTALL &
      PROC_PID=$!
  yum -y install yum-utils &> /dev/null 
  yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo &> /dev/null
  yum -y install device-mapper-persistent-data chrony lvm2 wget net-tools nfs-utils iptables-services lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo libaio-devel wget vim ncurses-devel autoconf automake zlib-devel openssh-server socat ipvsadm conntrack telnet iptables-services nginx-mod-stream &> /dev/null
      CHECK_STATUS ${PROC_PID}
  }
  
  function CHECK_FIREWALL_SELINUX_DATE(){
      PROC_NAME CHECK_FIREWALL_SELINUX_DATE &
      PROC_PID=$!
  systemctl stop firewalld &> /dev/null && systemctl disable firewalld &> /dev/null
  sed -i.bak 's/SELINUX=.*/SELINUX=disabled/g' /etc/sysconfig/selinux
  sed -i.bak 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
  setenforce 0 &> /dev/null
  if [ -z `egrep "ntp.aliyun.com" /etc/chrony.conf` ]; then
      sed -i.bak 'N;4 i server ntp.aliyun.com iburst' /etc/chrony.conf
  fi
  systemctl enable chronyd --now &> /dev/null
      CHECK_STATUS ${PROC_PID} 
  }
  
  function K8S_SYSCTL(){
      PROC_NAME K8S_SYSCTL &
      PROC_PID=$!
  cat > /etc/sysctl.d/k8s.conf <<'EOF'
  net.bridge.bridge-nf-call-ip6tables = 1
  net.bridge.bridge-nf-call-iptables = 1
  net.ipv4.ip_forward = 1
  EOF
  sysctl --system &> /dev/null
      CHECK_STATUS ${PROC_PID} 
  }
  
  function CHECK_SWAPOFF(){
      PROC_NAME CHECK_SWAPOFF &
      PROC_PID=$!
  if [[ ! `egrep "^#.*swap.*" /etc/fstab` ]]; then
  sed -i.bak 's/.*swap/#&/' /etc/fstab
  fi
  swapoff -a
  mount -a
      CHECK_STATUS ${PROC_PID} 
  }
  
  function CHECK_IPVS_NETFILTER(){
      PROC_NAME CHECK_IPVS_NETFILTER &
      PROC_PID=$!
  modprobe br_netfilter
  if [[ ! `egrep -o "modprobe br_netfilter" /etc/profile` ]]; then
  	echo "modprobe br_netfilter" >> /etc/profile
  fi
  cat > /etc/sysconfig/modules/ipvs.modules <<-'EOF'
  #!/bin/bash
  ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
  for kernel_module in ${ipvs_modules}; do
   /sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
   if [ 0 -eq 0 ]; then
   /sbin/modprobe ${kernel_module}
   fi
  done
  EOF
  chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
  systemctl stop iptables &> /dev/null && systemctl disable iptables &> /dev/null
  	CHECK_STATUS ${PROC_PID} 
  }
  
  function YUM_INSTATLL_DOCKER_ADD_DAEMONJSON(){
      PROC_NAME YUM_INSTATLL_DOCKER_ADD_DAEMONJSON &
      PROC_PID=$!
  yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo &> /dev/null
  yum install docker-ce -y &> /dev/null
  systemctl enable docker --now &> /dev/null
  cat > /etc/docker/daemon.json << 'EOF'
  {
  "registry-mirrors":["https://vh3bm52y.mirror.aliyuncs.com","https://registry.dockercn.com","https://docker.mirrors.ustc.edu.cn","https://dockerhub.azk8s.cn","http://hubmirror.c.163.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
  }
  EOF
  systemctl daemon-reload && systemctl restart docker && systemctl status docker --no-pager | egrep "Active: "
  	CHECK_STATUS ${PROC_PID} 
  }
  
  CHECK_SYSTEM 
  CHECK_USER
  CHECK_NETWORK
  ADD_K8S_REPO
  YUM_BASE_INSTALL
  CHECK_FIREWALL_SELINUX_DATE
  K8S_SYSCTL
  CHECK_SWAPOFF
  CHECK_IPVS_NETFILTER
  YUM_INSTATLL_DOCKER_ADD_DAEMONJSON

• 搭建 etcd 集群

  配置 etcd 工作目录

  在所有控制节点创建配置文件和证书文件存放目录

  mkdir -p /etc/etcd/ssl

  安装签发证书工具 cfssl

  mkdir /data/work -p && cd /data/work/

  从官方网站下载cfssl 工具

  证书 | Kuberneteshttps://kubernetes.io/zh/docs/tasks/administer-cluster/certificates/

•  下载 

  curl -L https://ghproxy.com/https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64 -o cfssl
  chmod +x cfssl
  curl -L https://ghproxy.com/https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64 -o cfssljson
  chmod +x cfssljson
  curl -L https://ghproxy.com/https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl-certinfo_1.5.0_linux_amd64 -o cfssl-certinfo
  chmod +x cfssl-certinfo

• 配置 ca 证书

  配置ca根 证书请求文件

  vim ca-csr.json

  {
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "Hubei",
        "L": "Wuhan",
        "O": "k8s",
        "OU": "system"
      }
    ],
    "ca": {
            "expiry": "87600h"
    }
  }

• 生成ca根证书

  cfssl gencert -initca ca-csr.json | cfssljson -bare ca

  配置ca根 证书配置文件ca-config.json  

  vim ca-config.json 

  {
    "signing": {
        "default": {
            "expiry": "87600h"
          },
        "profiles": {
            "kubernetes": {
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ],
                "expiry": "87600h"
            }
        }
    }
  }

• 配置 etcd 证书请求文件，host