CentOS 7下Kubernetes 1.16.4 + HAProxy + Keepalived 高可用集群安装

架构

Kubernetes高可用集群有两种部署方式，本文以第一种方式部署。

系统环境

cat /etc/redhat-release
CentOS Linux release 7.7.1908 (Core)
 
uname -a
Linux k8s-master 3.10.0-1062.9.1.el7.x86_64 #1 SMP Fri Dec 6 15:49:49 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux

vi /etc/hosts
192.168.1.60    vip.k8s.twingao.com    #虚拟IP地址，用于负载均衡，在三个主节点之间漂移。
192.168.1.61    k8s-master1            #主节点1 + HAProxy + Keepalived
192.168.1.62    k8s-master2            #主节点2 + HAProxy + Keepalived
192.168.1.63    k8s-master3            #主节点3 + HAProxy + Keepalived
192.168.1.64    k8s-node1              #工作节点1
192.168.1.65    k8s-node2              #工作节点2
192.168.1.66    k8s-node3              #工作节点3

关闭防火墙和安全设置。

systemctl stop firewalld
systemctl disable firewalld
 
vi /etc/fstab
#/dev/mapper/centos-swap swap                    swap    defaults        0 0
 
vi /etc/selinux/config
SELINUX=disabled
 
#重启生效
reboot

安装Keepalived

yum install -y keepalived

cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

vi /etc/keepalived/keepalived.conf
# k8s-maseter1
! Configuration File for keepalived

global_defs {
   router_id k8s-master1      #主机名
}

vrrp_instance VI_1 {
    state MASTER              #不用修改，主要按照优先级确定
    interface ens33           #vip所在的网卡
    virtual_router_id 51
    priority 150              #优先级，越大越优先
    advert_int 1              #MASTER与BACKUP同步通知间隔，单位为秒
    authentication {          #同一vrrp实例的MASTER与BACKUP使用相同的密码才能正常通信
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.60          #vip
    }
}

# k8s-maseter2
! Configuration File for keepalived

global_defs {
   router_id k8s-master2
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.60
    }
}

# k8s-maseter3
! Configuration File for keepalived

global_defs {
   router_id k8s-master3
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.60
    }
}

systemctl start keepalived
systemctl enable keepalived

由于k8s-master1节点的优先级最高，vip应该在k8s-master1，其它节点没有vip。

ip addr show ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:3d:77:8f brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.61/24 brd 192.168.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.1.60/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe3d:778f/64 scope link
       valid_lft forever preferred_lft forever

安装HAProxy

yum install -y haproxy

cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

vi /etc/haproxy/haproxy.cfg
#三个master节点的配置相同。
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    tcp        #改为tcp
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  main *:8443             #改为8443，为HAProxy监听客户端的端口。
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js

    use_backend static          if url_static
    default_backend             k8s        #改为k8s，与backend k8s对应

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
    balance     roundrobin
    server      static 127.0.0.1:4331 check

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend k8s            #负载均衡k8s的api-server，对应三个k8s-master节点。
    balance     roundrobin
    server k8s-master1 192.168.1.61:6443 check
    server k8s-master2 192.168.1.62:6443 check
    server k8s-master3 192.168.1.63:6443 check

systemctl start haproxy
systemctl enable haproxy

安装Kubernetes

修改内核参数。

cat <<EOF> /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
 
modprobe br_netfilter
sysctl --system

* Applying /usr/lib/sysctl.d/00-system.conf ...
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
* Applying /usr/lib/sysctl.d/10-default-yama-scope.conf ...
kernel.yama.ptrace_scope = 0
* Applying /usr/lib/sysctl.d/50-default.conf ...
kernel.sysrq = 16
kernel.core_uses_pid = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.all.promote_secondaries = 1
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
* Applying /etc/sysctl.d/99-sysctl.conf ...
* Applying /etc/sysctl.d/k8s.conf ...
#注意需要有以下两行
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
* Applying /etc/sysctl.conf ...

安装Docker。

yum install -y yum-utils device-mapper-persistent-data lvm2
 
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
或者
#yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
 
yum list docker-ce --showduplicates | sort -r
已加载插件：fastestmirror
可安装的软件包
 * updates: mirrors.huaweicloud.com
Loading mirror speeds from cached hostfile
 * extras: mirror.bit.edu.cn
docker-ce.x86_64            3:19.03.5-3.el7                     docker-ce-stable
docker-ce.x86_64            3:19.03.4-3.el7                     docker-ce-stable
docker-ce.x86_64            3:19.03.3-3.el7                     docker-ce-stable
docker-ce.x86_64            3:19.03.2-3.el7                     docker-ce-stable
docker-ce.x86_64            3:19.03.1-3.el7                     docker-ce-stable
docker-ce.x86_64