使用kubeadm搭建k8s高可用集群

环境准备

系统使用的Ubuntu18.04

主机IP	主机名	docker版本
172.31.1.10	k8s-master1	19.03.15
172.31.1.11	k8s-master2	19.03.15
172.31.1.12	k8s-master3	19.03.15
172.31.1.13	harbor	19.03.15
172.31.1.14	haproxy1
172.31.1.15	haproxy2
172.31.1.16	k8s-node1	19.03.15
172.31.1.17	k8s-node2	19.03.15
172.31.1.18	k8s-node3	19.03.15

改主机名，因为k8s是以主机名区分的

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-master1.example.local

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-master2.example.local

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-master3.example.local

root@k8s-ubuntu:~# hostnamectl set-hostname harbor.example.local

root@k8s-ubuntu:~# hostnamectl set-hostname ha1.example.local

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-node1.example.local

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-node2.example.local

[root@long-ubuntu ~]# hostnamectl set-hostname k8s-node3.example.local

Ubuntu1804一键安装docker-ce

#!/bin/bash
# Ubuntu Install docker-ce

apt purge ufw lxd lxd-client lxcfs -y lxc-common

apt install -y iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common  \
lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet  \
traceroute gcc openssh-server lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev \
zlib1g-dev  ntpdate tcpdump telnet traceroute iotop unzip zip

apt-get remove docker docker-engine docker.io

apt-get install -y apt-transport-https ca-certificates curl gnupg2 software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

sudo add-apt-repository    "deb [arch=amd64] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu \
  $(lsb_release -cs) \
  stable"

apt update

apt install -y docker-ce=5:19.03.15~3-0~ubuntu-bionic docker-ce-cli=5:19.03.15~3-0~ubuntu-bionic

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://rzd1bb7q.mirror.aliyuncs.com"]
}
EOF

sudo systemctl daemon-reload
sudo systemctl restart docker
docker version

记得关闭swap

关闭防火墙

优化内核参数

[root@long ~]# sysctl -a | grep forward
net.ipv4.ip_forward = 1

[root@long ~]# sysctl -a | grep bridge-nf-call
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

keepalived + haproxy 安装

# 172.31.1.14
[root@ha1 ~]# apt -y install keepalived haproxy

配置keepalived

[root@ha1 ~]# find / -name "*keepalived*"

# 拷贝
[root@ha1 ~]# cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf

测试ip是有被使用

[root@k8s-master1 ~]# ping 172.31.1.188
PING 172.31.1.188 (172.31.1.188) 56(84) bytes of data.
From 172.31.1.10 icmp_seq=1 Destination Host Unreachable
From 172.31.1.10 icmp_seq=2 Destination Host Unreachable
From 172.31.1.10 icmp_seq=3 Destination Host Unreachable

# 上面提示就是没有，所以以下可以设置成VIP的ip地址

修改配置

[root@ha1 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    garp_master_delay 10
    smtp_alert
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.31.1.188 dev eth0 label eth0:1
    }
}

开机启动

[root@ha1 ~]# systemctl enable --now keepalived

查看

[root@ha1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:da:36:40 brd ff:ff:ff:ff:ff:ff
    inet 172.31.1.14/21 brd 172.31.7.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 172.31.1.188/32 scope global eth0:1
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feda:3640/64 scope link
       valid_lft forever preferred_lft forever

配置HAproxy

[root@ha1 ~]# vim /etc/haproxy/haproxy.cfg

listen stats
  mode http
  bind 0.0.0.0:9999
  stats enable
  log global
  stats uri /haproxy-status
  stats auth haadmin:123456

listen k8s-m44-6443
  bind 172.31.1.188:6443
  mode tcp
  server 172.31.1.10 172.31.1.10:6443 check inter 2s fall 3 rise 5
  server 172.31.1.11 172.31.1.11:6443 check inter 2s fall 3 rise 5
  server 172.31.1.12 172.31.1.12:6443 check inter 2s fall 3 rise 5

开机启动

[root@ha1 ~]# systemctl enable --now haproxy
Synchronizing state of haproxy.service with SysV service s