ansible-部署-k8s-docker-haproxy

实验环境:机器性能好的话可以做成高可用
10.0.0.142 k8s-master
10.0.0.152 harbor
10.0.0.162 etcd
10.0.0.172 node1
10.0.0.182 node2
10.0.0.192 harpoxy
10.0.0.202 harpoxy

apt-get update && apt-get upgrade -y && apt-get dist-upgrade -y

安装python2

apt-get install python2.7

Ubuntu16.04可能需要配置以下软连接

ln -s /usr/bin/python2.7 /usr/bin/python

在控制端安装ansible

root@k8s:~# apt install python-pip -y
root@k8s:~# pip install ansible

生成秘钥对

root@k8s:~# ssh-keygen
root@k8s:~# apt-get install sshpass

编写公钥分发脚本

root@k8s:~# cat scp.sh 
#!/vin/bash

HOST="
10.0.0.142
10.0.0.152
10.0.0.162
10.0.0.172
10.0.0.182
10.0.0.192
"
for node in ${HOST};do
 sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
 if [ $? -eq 0 ];then
   echo "${node} 秘钥copy完成"
 else
   echo "${node} 秘钥copy失败"
 fi
done

执行脚本完成后测试是否成功

root@k8s:~# ssh 10.0.0.152

安装docker

root@harbor:/usr/local/src# cat docker-install.sh 
# step 1: 安装必要的一些系统工具
sudo apt-get update
sudo apt-get -y install apt-transport-https ca-certificates curl software-properties-common
# step 2: 安装GPG证书
curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
# Step 3: 写入软件源信息
sudo add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
# Step 4: 更新并安装 Docker-CE
sudo apt-get -y update
sudo apt-get -y install docker-ce docker-ce-cli

配置镜像加速器

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://gbxguec2.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
root@harbor:/usr/local/src# apt install docker-compose

部署镜像仓库harbor

root@harbor:~# cd /usr/local/src
root@harbor:/usr/local/src# ls
harbor-offline-installer-v1.7.6.tgz
root@harbor:/usr/local/src# tar xvf harbor-offline-installer-v1.7.6.tgz

root@harbor:/usr/local/src# cd harbor/
root@harbor:/usr/local/src/harbor# vim harbor.cfg #修改harbor配置文件
hostname = harbor.linux.local #没有域名需要自己配置hosts解析
ui_url_protocol = https #协议这里用https
ssl_cert =  /usr/local/src/harbor/cetrs/harbor-ca.crt #证书路径没有需要自己创建
ssl_cert_key = /usr/local/src/harbor/cetrs/harbor-ca.key
email_server = smtp.mydomain.com #邮件可以用自己的邮箱
email_server_port = 25
email_username = sample_admin@mydomain.com
email_password = abc
email_from = admin <sample_admin@mydomain.com>
email_ssl = false
email_insecure = false
创建证书目录
root@harbor:/usr/local/src/harbor# mkdir cetrs
root@harbor:/usr/local/src/harbor# cd cetrs/
#生成私有key
root@harbor:/usr/local/src/harbor/cetrs# openssl genrsa -out /usr/local/src/harbor/cetrs/harbor-ca.key

签发证书
root@harbor:/usr/local/src/harbor/cetrs# openssl req -x509 -new -nodes -key /usr/local/src/harbor/cetrs/harbor-ca.key -subj "/CN=harbor.linux.local" -days 7120 -out /usr/local/src/harbor/cetrs/harbor-ca.crt

以上步骤完成后执行以下命令

root@harbor:/usr/local/src/harbor# ./install.sh 

因为我们没有dns所以需要在windows加解析加完之后访问https://harbor.linux.local/

client同步证书：在哪个服务器上传镜像就在哪个服务器手动添加证书

root@k8s:~# mkdir /etc/docker/certs.d/harbor.linux.local -p #创建以harbor域名命名的目录
root@k8s:~# cd /etc/docker/certs.d/harbor.linux.local

在harbor服务器把证书拷贝到上传镜像那台服务器

root@harbor:/usr/local/src/harbor/cetrs# scp harbor-ca.crt 10.0.0.142:/etc/docker/certs.d/harbor.linux.local

k8s-master服务器安装docker 正常情况应该用ansible来安装docker

root@harbor:/usr/local/src# scp docker-install.sh 10.0.0.142:/root
root@k8s:~# bash docker-install.sh

在10.0.0.142装完docker之后在服务器写上harbor的hosts解析测试harbor登录

root@k8s:~# vim /etc/hosts
10.0.0.152 harbor.linux.local
测试登录
root@k8s:~# docker login harbor.linux.local
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

配置镜像加速器

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://gbxguec2.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

去docker官网pull一个centos镜像
https://hub.docker.com/_/centos?tab=tags&page=1&ordering=last_updated

root@k8s:~# docker pull centos:7.8.2003
root@k8s:~# docker images
REPOSITORY   TAG        IMAGE ID       CREATED         SIZE
centos       7.8.2003   afb6fca791e0   12 months ago   203MB

修改镜像的tag上传到镜像仓库

root@k8s:~# docker tag centos:7.8.2003 harbor.linux.local/baseimages/centos:7.8.2003

去harbor创建一个新的仓库项目

root@k8s:~# docker push centos:harbor.linux.local/baseimages/centos:centos7.8.2003

给其他机器分发公钥以及hosts文件等

root@k8s:~# cat scp.sh 
#!/bin/bash

HOST="
10.0.0.152
10.0.0.162
10.0.0.172
10.0.0.182
10.0.0.192
"
for node in ${HOST};do
# sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
 #if [ $? -eq 0 ];then
 # echo "${node} Copy complete"
 #else
 # echo "${node} Copies of the failure"
 #fi
  ssh ${node} "mkdir /etc/docker/certs.d/harbor.linux.local -p"
  echo "Certificate directory created successfully"
  scp /etc/docker/certs.d/harbor.linux.local/harbor-ca.crt ${node}:/etc/docker/certs.d/harbor.linux.local/harbor-ca.crt
  echo "Copy complete"
  ssh ${node} "echo "10.0.0.152 harbor.linux.local" >> /etc/hosts"
  echo "host file copy comlete"
  
  
done

在ansible控制端编排k8s安装
（项目安装来自https://github.com/easzlab/kubeasz）

root@k8s:~# export release=2.2.1
root@k8s:~# echo $release

下载项目脚本

root@k8s:~# curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/easzup
root@k8s:~# chmod a+x easzup 
root@k8s:~# ./easzup --help
./easzup: illegal option -- -
Usage: easzup [options] [args]
  option: -{DdekSz}
    -C         stop&clean all local containers
    -D         download all into /etc/ansible
    -P         download system packages for offline installing
    -S         start kubeasz in a container
    -d <ver>   set docker-ce version, default "19.03.8"
    -e <ver>   set kubeasz-ext-bin version, default "0.5.2"
    -k <ver>   set kubeasz-k8s-bin version, default "v1.18.2"
    -m <str>   set docker registry mirrors, default "CN"(used in Mainland,China)
    -p <ver>   set kubeasz-sys-pkg version, default "0.3.3"
    -z <ver>   set kubeasz version, default "2.2.1"
  
see more at https://github.com/kubeasz/dockerfiles
执行以下命令下载所需的二进制包，默认是放在/etc/ansible
root@k8s:~# ./easzup -D

下载完成以后yaml文件会拷贝到以下目录


配置集群参数

cd /etc/ansible && cp example/hosts.multi-node hosts, 然后实际情况修改此hosts文件

root@k8s:~# cd /etc/ansible/
root@k8s:/etc/ansible# cp example/hosts.
hosts.allinone    hosts.multi-node  
root@k8s:/etc/ansible# cp example/hosts.multi-node ./hosts #ansible实际读取的文件

接下来根据实际情况修改hosts文件，有些配置保持默认

[etcd]
#192.168.1.1 NODE_NAME=etcd1
#192.168.1.2 NODE_NAME=etcd2
10.0.0.162 NODE_NAME=etcd1
[kube-master]
#192.168.1.1
#192.168.1.2
10.0.0.142
[kube-node]
#192.168.1.3
#192.168.1.4
10.0.0.172
#10.0.0.182

[ex-lb]
10.0.0.192 LB_ROLE=backup EX_APISERVER_VIP=10.0.0.212 EX_APISERVER_PORT=6443
10.0.0.202 LB_ROLE=master EX_APISERVER_VIP=10.0.0.212 EX_APISERVER_PORT=6443
ONTAINER_RUNTIME="docker"

# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"#网络组件

# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"

# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="192.168.0.0/16" #service 网段

# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="172.20.0.0/16" #pod网段  注意三个网段都不能冲突

# NodePort Range
NODE_PORT_RANGE="20000-40000"

# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local. #域名后缀

安装keepalived

apt install keepalived
root@ubuntu1804:/etc/keepalived# find / -name *.vrrp
root@ubuntu1804:/etc/keepalived# cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf
root@ubuntu1804:/etc/keepalived# vim keepalived.conf
vrrp_instance VI_1 {
    state MASTER  
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 99
    priority 100  
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
    10.0.0.212 dev eth0 label eth0:0
    10.0.0.213 dev eth0 label eth0:1
    }
}

另一台keepalived

 apt install keepalived
root@ubuntu1804:/etc/keepalived# find / -name *.vrrp
root@ubuntu1804:/etc/keepalived# cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf
root@ubuntu1804:/etc/keepalived# vim keepalived.conf
vrrp_instance VI_1 {
    state BACKUP #调整
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 99
    priority 80  #调整优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
    10.0.0.212 dev eth0 label eth0:0
    10.0.0.213 dev eth0 label eth0:1
    }
}

编译安装haproxy
内核参数一定要修改

net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1

安装基础命令及编译依赖环境

apt install gcc iproute2 ntpdate tcpdump telnet traceroute  nfs-kernel-server  nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev openssh-server lib
readline-dev libsystemd-dev

root@ubuntu1804:~# cd /usr/local/src/
root@ubuntu1804:/usr/local/src#  wget http://www.lua.org/ftp/lua-5.3.5.tar.gz
root@ubuntu1804:/usr/local/src# tar xvf lua-5.3.5.tar.gz
# cd lua-5.3.5
# make linux test
root@ubuntu1804:/usr/local/src/lua-5.3.5# ./src/lua -v
Lua 5.3.5  Copyright (C) 1994-2018 Lua.org, PUC-Rio

root@ubuntu1804:/usr/local/src# cd haproxy-2.1.4/

#make ARCH=x86_64 TARGET=linux-glibc USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_LUA=1 LUA_INC=/usr/local/src/lua-5.3.5/src/ LUA_LIB=/usr/local/src/lua-5.3.5/src/

make install PREFIX=/apps/haproxy
ln -s /apps/haproxy/sbin/haproxy /usr/sbin/

验证版本

root@ubuntu1804:/usr/local/src/haproxy-2.1.4# haproxy -v
HA-Proxy version 2.1.4 2020/04/02 - https://haproxy.org/

创建haproxy的启动脚本

root@ubuntu1804:/usr/local/src/haproxy-2.1.4# cat /lib/systemd/system/haproxy.service 
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q 
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID
[Install]
WantedBy=multi-user.target

创建自定配置文件

mkdir /etc/haproxy
 vim /etc/haproxy/haproxy.cfg
 global
    maxconn 100000
    chroot /apps/haproxy
    stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
    #uid 99
    #gid 99
    user haproxy
    group haproxy
    daemon
    #nbproc 4
    #cpu-map 1 0
    #cpu-map 2 1
    #cpu-map 3 2
    #cpu-map 4 3
    pidfile /var/lib/haproxy/haproxy.pid
    log 127.0.0.1 local2 info
defaults 
   option http-keep-alive
   option forwardfor
   maxconn 100000
   mode http
   timeout connect 300000ms 
   timeout client 300000ms
   timeout server 300000ms
listen stats
   mode http
   bind 0.0.0.0:9999 
   stats enable
   log global
   stats uri /haproxy-status
   stats auth haadmin:123456 
 listen k8s-api-6443
   bind 10.0.0.212:6443
   mode tcp
   server 10.0.0.142 10.0.0.142:6443 check inter 3000 fall 2 rise 5

启动haproxy

 mkdir /var/lib/haproxy
 chown -R 99.99 /var/lib/haproxy/
 useradd -r -s /sbin/nologin -d /var/lib/haproxy haproxy
 systemctl daemon-reload
 systemctl enable --now haproxy

浏览器访问状态页

接下来测试环境

root@k8s:/etc/ansible# ansible all -m ping

测试成功后进行预检查和初始由于我们负载均衡和时间同已经做了需要做下修改

root@k8s:/etc/ansible# vim 01.prepare.ym

接下来就可以去执行ansible脚本部署k8s了
脚本必须从01开始执行

root@k8s:/etc/ansible# ansible-playbook 01.prepare.yml 

安装etcd执行02.etcd.yml

安装docker

root@k8s:/etc/ansible# ansible-playbook 03.docker.yml

检查node节点装完docker能否拉取镜像

在node节点拉取镜像

root@node1:~# docker pull harbor.linux.local/baseimages/centos:centos7.8.2003

安装k8s-master

root@k8s:/etc/ansible# ansible-playbook 04.kube-master.yml 

root@k8s:/etc/ansible# vim /root/.kube/config  #这个路径指定了VIP端口
把地址修改成咋们的VIP地址

安装node节点

root@k8s:/etc/ansible# ansible-playbook 05.kube-node.yml 

安装k8s的网络组件
因为镜像是在公网上下载比较慢，下载下来上传到自己的harbor仓库

root@k8s:/etc/ansible# docker tag calico/cni:v3.8.8-1 harbor.linux.local/baseimages/cni:v3.8.8-1
root@k8s:/etc/ansible# docker push harbor.linux.local/baseimages/cni:v3.8.8-1

root@k8s:/etc/ansible# vim roles/calico/templates/calico-v3.8.yaml.j2 #在这个文件里修改镜像下载路径

root@k8s:/etc/ansible# docker tag calico/pod2daemon-flexvol:v3.8.8 harbor.linux.local/baseimages/pod2daemon-flexvol:v3.8.8
root@k8s:/etc/ansible# docker push harbor.linux.local/baseimages/pod2daemon-flexvol:v3.8.8

root@k8s:~# docker tag calico/node:v3.8.8-1 harbor.linux.local/baseimages/node:v3.8.8-1
root@k8s:~# docker push harbor.linux.local/baseimages/node:v3.8.8-1

root@k8s:~# docker tag calico/kube-controllers:v3.8.8 harbor.linux.local/baseimages/kube-controllers:v3.8.8
root@k8s:~# docker push harbor.linux.local/baseimages/kube-controllers:v3.8.8

以上修改完之后执行

root@k8s:/etc/ansible# ansible-playbook 06.network.yml 

验证

root@k8s:/etc/ansible# calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+--------------+-------------------+-------+----------+-------------+
| 10.0.0.172   | node-to-node mesh | up    | 11:11:42 | Established |
+--------------+-------------------+-------+----------+-------------+

创建dashbord

root@k8s:/etc/ansible/manifests/dashboard# cd /etc/ansible/manifests/dashboard/
root@k8s:/etc/ansible/manifests/dashboard# mkdir 2.0.1

上传dashbord文件

root@k8s:/etc/ansible/manifests/dashboard/2.0.1# vim dashboard-2.0.1-magedu.yml 

root@k8s:~# docker pull kubernetesui/dashboard:v2.0.1
root@k8s:~# docker tag kubernetesui/dashboard:v2.0.1 harbor.linux.local/baseimages/dashboard:v2.0.1
root@k8s:~# docker tag kubernetesui/dashboard:v2.0.1 harbor.linux.local/baseimages/dashboard:v2.0.1
root@k8s:/etc/ansible/manifests/dashboard/2.0.1# vim dashboard-2.0.1-magedu.yml

root@k8s:~# docker pull kubernetesui/metrics-scraper:v1.0.4
root@k8s:~# docker tag kubernetesui/metrics-scraper:v1.0.4 harbor.linux.local/baseimages/metrics-scraper:v1.0.4
root@k8s:/etc/ansible/manifests/dashboard/2.0.1# vim dashboard-2.0.1-magedu.yml

修改完成之后执行

root@k8s:/etc/ansible/manifests/dashboard/2.0.1# kubectl apply -f dashboard-2.0.1-magedu.yml 

root@k8s:/etc/ansible/manifests/dashboard/2.0.1# kubectl apply -f admin-user.yml

检查dashbord是否部署完成

root@k8s:/etc/ansible/manifests/dashboard/2.0.1# kubectl get pod -A

检查node节点是否监听30002端口

由于时间关系，暂时到这里，后续在补充