k8s-kubeasz部署集群打怪升级之路X

#k8s测试集群

HAproxy-VIP 192.168.149.20
master1   192.168.149.22
master2   192.168.149.23
harbor1   192.168.149.24
harbor2   192.168.149.25
etcd1       192.168149.26
etcd2       192.168149.27
etcd3       192.168149.28
haproxy1 192.168.149.29
haproxy2 192.168.149.30
node1      192.168.149.31
node2      192.168.149.32

 思路：1.环境初始化->2.部署etcd集群->3.部署docker->4.部署master->5.部署node->6.部署网络插件flannel

后续一篇在介绍添加master节点与node节点步骤

注意！！！！

kubeasz 3.0之前存在ansible group命名不规范问题（group 'kube-node' --> group 'kube_node'，所以部署kubeasz 2.x版本时需要

grep -i kube-master . -R |awk -F: '{print $1}'|grep -v easzctl|xargs sed -i 's/kube-master/kube_master/g'
grep -i kube-node . -R |awk -F: '{print $1}'|grep -v easzctl|xargs sed -i 's/kube-node/kube_node/g'
grep -i ex-lb . -R |awk -F: '{print $1}'|grep -v easzctl|xargs sed -i 's/ex-lb/ex_lb/g'

注意不要将yaml下的role也一同改了，需要改的地方是yaml的hosts段

04.kube-master.yml    05.kube-node.yml   ---> roles 

使用的是python2.7版本，pip也需要安装

版本信息 cat /root/kubeasz-2.2.4/tools/easzup

DOCKER_VER=19.03.14
KUBEASZ_VER=2.2.4
K8S_BIN_VER=v1.20.2
EXT_BIN_VER=0.8.1
SYS_PKG_VER=0.3.3

# images needed by k8s cluster
calicoVer=v3.15.3
corednsVer=1.7.1
dashboardVer=v2.1.0
dashboardMetricsScraperVer=v1.0.6
flannelVer=v0.13.0-amd64
metricsVer=v0.3.6
pauseVer=3.2
traefikVer=v2.4

-----------------------------------------------------k8s准备部署环境-----------------------------------------------------

-----------------------------------【准备环境】-------------------------------------
--- ansible免密登录
[root@master1 ~]# cat hosts.list 
192.168.149.22
192.168.149.23
192.168.149.24
192.168.149.25
192.168.149.26
192.168.149.27
192.168.149.28
192.168.149.29
192.168.149.30
192.168.149.31
192.168.149.32
[root@master1 ~]# cat push_ssh_key.sh 
#!/bin/bash
rpm -q sshpass &> /dev/null || yum -y install sshpass
[ -f /root/.ssh/id_rsa  ] || ssh-keygen -f /root/.ssh/.id_rsa -P ''
export SSHPASS=123456
while read IP;do
   sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $IP
done < hosts.list

--- 主机名hostname
[root@master1 ~]# cat scripts/change_hostname.sh
#!/bin/bash
Name=`hostname`
hostnamectl set-hostname ${Name}.yyy.com
hostname
[root@master1 ~]# cat scripts/change_hostname.yaml 
---
- hosts: all
  remote_user: root
  gather_facts: no

  tasks:
  - name: copy sh 
    copy: src=/root/scripts/change_hostname.sh dest=/root/
  - name: exec sh
    shell: sh /root/change_hostname.sh

--- 下载kubeasz并复制hosts模板文件
git clone https://github.com/easzlab/kubeasz/archive/refs/tags/3.0.1.tar.gz

或者直接在github上下载然后在服务器上解压

[root@master1 example]# cp /root/kubeasz-2.2.4/example/hosts.multi-node /etc/ansible/hosts


--- hosts配置文件
[root@master1 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.149.24 harbor.yyy.com

192.168.149.31 node1.yyy.com
192.168.149.32 node2.yyy.com
192.168.149.29 haproxy1.yyy.com
192.168.149.25 harbor2.yyy.com
192.168.149.30 haproxy2.yyy.com
192.168.149.26 etcd1.yyy.com
192.168.149.27 etcd2.yyy.com
192.168.149.22 master1.yyy.com
192.168.149.28 etcd3.yyy.com
192.168.149.23 master2.yyy.com

--- ansible主控端inventory的hosts
[root@master1 ~]# cat /etc/ansible/hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster
[etcd]
192.168.149.26 NODE_NAME=etcd1
192.168.149.27 NODE_NAME=etcd2
192.168.149.28 NODE_NAME=etcd3

# master node(s)
[kube_master]
192.168.149.22
192.168.149.23

# work node(s)
[kube_node]
192.168.149.31
192.168.149.32

[other]
192.168.149.24
192.168.149.25
192.168.149.29
192.168.149.30

# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one
# 'SELF_SIGNED_CERT': 'no' you need put files of certificates named harbor.pem and harbor-key.pem in directory 'down'
[harbor]

  
[harbor]
#192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no SELF_SIGNED_CERT=yes

# [optional] loadbalance for accessing k8s from outside
#【可选】外部负载均衡，用于自有环境负载转发 NodePort 暴露的服务等
[ex_lb]                                           
#192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443

# [optional] ntp server for the cluster
[chrony]
#192.168.1.1

[all:vars]
# --------- Main Variables ---------------
# Cluster container-runtime supported: docker, containerd
CONTAINER_RUNTIME="docker"

# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="flannel"            ##网络插件 flannel

# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"                  #kube-proxy代理模式

# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.68.0.0/16"    # 服务网段 (Service CIDR），注意不要与内网已有网段冲突  

# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="172.20.0.0/16"     # POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突

# NodePort Range                  
NODE_PORT_RANGE="20000-40000"             # 服务端口范围 (NodePort Range)

# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="yyy.local."

# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/opt/kube/bin"

# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"

# Deploy Directory (kubeasz workspace)
base_dir="/etc/ansible"

 -----------------------------------------------------k8s准备部署环境----------------------------------------------------

===============【开始初始化】==============

1.    01.prepare.yml-创建证书和环境准备

  [root@master1 kubeasz-2.2.4]# ansible-playbook 01.prepare.yml     

• role:deploy，创建CA证书、集群组件访问apiserver所需的各种kubeconfig
• role:prepare，系统基础环境配置、分发CA证书、kubectl客户端安装

[root@master1 kubeasz-2.2.4]# cat 01.prepare.yml 
# [optional] to synchronize system time of nodes with 'chrony' 
- hosts:
  - kube_master
  - kube_node
  - etcd
  - ex_lb
  - chrony
  roles:
  - { role: chrony, when: "groups['chrony']|length > 0" }

# to create CA, kubeconfig, kube-proxy.kubeconfig etc.
- hosts: localhost
  roles:
  - deploy

# prepare tasks for all nodes
- hosts:
  - kube_master
  - kube_node
  - etcd
  roles:
  - prepare

执行过程

[root@master1 kubeasz-2.2.4]# ansible-playbook 01.prepare.yml     

PLAY [kube_master,kube_node,etcd,ex_lb,chrony] ************************************************************************************************************************

TASK [Gathering Facts] ************************************************************************************************************************************************
ok: [192.168.149.22]
ok: [192.168.149.31]
ok: [192.168.149.32]
ok: [192.168.149.26]
ok: [192.168.149.23]
ok: [192.168.149.27]
ok: [192.168.149.28]

PLAY [localhost] ******************************************************************************************************************************************************

TASK [Gathering Facts] ************************************************************************************************************************************************
ok: [localhost]

TASK [deploy : prepare some dirs] *************************************************************************************************************************************
ok: [localhost] => (item=/etc/ansible/.cluster/ssl)
ok: [localhost] => (item=/etc/ansible/.cluster/backup)

TASK [deploy : 本地设置 bin 目录权限] *****************************************************************************************************************************************
ok: [localhost]

TASK [deploy : 读取ca证书stat信息] ******************************************************************************************************************************************
ok: [localhost]

TASK [deploy : 删除原有kubeconfig] ****************************************************************************************************************************************
ok: [localhost]

TASK [deploy : 准备kubectl使用的admin证书签名请求] *******************************************************************************************************************************
ok: [localhost]

TASK [deploy : 创建admin证书与私钥] ******************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置集群参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置客户端认证参数] *********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置上下文参数] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 选择默认上下文] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 准备kube-proxy 证书签名请求] ***********************************************************************************************************************************
ok: [localhost]

TASK [deploy : 创建 kube-proxy证书与私钥] ************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置集群参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置客户端认证参数] *********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置上下文参数] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 选择默认上下文] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 准备kube-controller-manager 证书签名请求] **********************************************************************************************************************
ok: [localhost]

TASK [deploy : 创建 kube-controller-manager证书与私钥] ***********************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置集群参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置认证参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置上下文参数] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 选择默认上下文] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 准备kube-scheduler 证书签名请求] *******************************************************************************************************************************
ok: [localhost]

TASK [deploy : 创建 kube-scheduler证书与私钥] ********************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置集群参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置认证参数] ************************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 设置上下文参数] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 选择默认上下文] ***********************************************************************************************************************************************
changed: [localhost]

TASK [deploy : 本地创建 easzctl 工具的软连接] ***********************************************************************************************************************************
ok: [localhost]

TASK [deploy : ansible 控制端创建 kubectl 软链接] *****************************************************************************************************************************
ok: [localhost]

TASK [deploy : 注册变量以判断是否容器化运行ansible控制端] ******************************************************************************************************************************
changed: [localhost]

TASK [deploy : ansible 控制端写入环境变量$PATH] ********************************************************************************************************************************
changed: [localhost]

TASK [deploy : ansible 控制端添加 kubectl 自动补全] ****************************************************************************************************************************
changed: [localhost]

TASK [deploy : pip install netaddr] ***********************************************************************************************************************************
fatal: [localhost]: FAILED! => {"changed": false, "msg": "Unable to find any of pip2, pip to use.  pip needs to be installed."}
...ignoring

PLAY [kube_master,kube_node,etcd] *************************************************************************************************************************************

TASK [prepare : 删除centos/redhat默认安装] **********************************************************************************************************************************
changed: [192.168.149.22] => (item=firewalld)
changed: [192.168.149.22] => (item=python-firewall)
changed: [192.168.149.22] => (item=firewalld-filesystem)
changed: [192.168.149.31] => (item=firewalld)
changed: [192.168.149.32] => (item=firewalld)
changed: [192.168.149.31] => (item=python-firewall)
changed: [192.168.149.32] => (item=python-firewall)
changed: [192.168.149.26] => (item=firewalld)
changed: [192.168.149.31] => (item=firewalld-filesystem)
changed: [192.168.149.32] => (item=firewalld-filesystem)
changed: [192.168.149.26] => (item=python-firewall)
changed: [192.16