CC00028.CloudKubernetes——|Kubernetes&高可用集群.V02|——|Keepalived&haproxy|

一、 所有master节点部署keepalived

### --- 安装相关包和keepalived

[root@k8s-master1 ~]# yum install -y conntrack-tools libseccomp libtool-ltdl
[root@k8s-master1 ~]# yum install -y keepalived
[root@k8s-master2 ~]# yum install -y conntrack-tools libseccomp libtool-ltdl
[root@k8s-master2 ~]# yum install -y keepalived

### --- 配置master节点
~~~     master1节点配置

[root@k8s-master1 ~]# cat > /etc/keepalived/keepalived.conf <<EOF 
> ! Configuration File for keepalived
> 
> global_defs {
>    router_id k8s
> }
> 
> vrrp_script check_haproxy {
>     script "killall -0 haproxy"
>     interval 3
>     weight -2
>     fall 10
>     rise 2
> }
> 
> vrrp_instance VI_1 {
>     state MASTER 
>     interface ens34 
>     virtual_router_id 51
>     priority 250
>     advert_int 1
>     authentication {
>         auth_type PASS
>         auth_pass ceb1b3ec013d66163d6ab
>     }
>     virtual_ipaddress {
>         10.10.10.15
>     }
>     track_script {
>         check_haproxy
>     }
> 
> }
> EOF

~~~     master2节点配置

[root@k8s-master2 ~]# cat > /etc/keepalived/keepalived.conf <<EOF 
> ! Configuration File for keepalived
> 
> global_defs {
>    router_id k8s
> }
> 
> vrrp_script check_haproxy {
>     script "killall -0 haproxy"
>     interval 3
>     weight -2
>     fall 10
>     rise 2
> }
> 
> vrrp_instance VI_1 {
>     state MASTER 
>     interface ens34 
>     virtual_router_id 51
>     priority 250
>     advert_int 1
>     authentication {
>         auth_type PASS
>         auth_pass ceb1b3ec013d66163d6ab
>     }
>     virtual_ipaddress {
>         10.10.10.15
>     }
>     track_script {
>         check_haproxy
>     }
> 
> }
> EOF

### --- 启动和检查
~~~     在两台master节点都执行
~~~     启动keepalived

[root@k8s-master1 ~]# systemctl start keepalived.service
[root@k8s-master2 ~]# systemctl start keepalived.service

~~~     设置开机启动

[root@k8s-master1 ~]# systemctl enable keepalived.service
[root@k8s-master2 ~]# systemctl enable keepalived.service

~~~     查看启动状态

[root@k8s-master1 ~]# systemctl status keepalived.service
[root@k8s-master2 ~]# systemctl status keepalived.service

~~~     启动后查看master1的网卡信息
~~~     目前在k8s-master2上，当k8s-master挂掉会漂移到k8s-master1上

[root@k8s-master2 ~]# ip a s ens34          
3: ens34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    inet 10.10.10.12/24 brd 10.10.10.255 scope global noprefixroute ens34
    inet 10.10.10.15/32 scope global ens34
       valid_lft forever preferred_lft forever

二、部署haproxy(所有master节点上部署)

### --- 安装

[root@k8s-master1 ~]# yum install -y haproxy
[root@k8s-master2 ~]# yum install -y haproxy

### --- 两台master节点的配置均相同，配置中声明了后端代理的两个master节点服务器，
~~~     指定了haproxy运行的端口为16443等，因此16443端口为集群的入口

[root@k8s-master1 ~]# cat > /etc/haproxy/haproxy.cfg << EOF
[root@k8s-master2 ~]# cat > /etc/haproxy/haproxy.cfg << EOF
> #---------------------------------------------------------------------
> # Global settings
> #---------------------------------------------------------------------
> global
>     # to have these messages end up in /var/log/haproxy.log you will
>     # need to:
>     # 1) configure syslog to accept network log events.  This is done
>     #    by adding the '-r' option to the SYSLOGD_OPTIONS in
>     #    /etc/sysconfig/syslog
>     # 2) configure local2 events to go to the /var/log/haproxy.log
>     #   file. A line like the following can be added to
>     #   /etc/sysconfig/syslog
>     #
>     #    local2.*                       /var/log/haproxy.log
>     #
>     log         127.0.0.1 local2
>     
>     chroot      /var/lib/haproxy
>     pidfile     /var/run/haproxy.pid
>     maxconn     4000
>     user        haproxy
>     group       haproxy
>     daemon 
>        
>     # turn on stats unix socket
>     stats socket /var/lib/haproxy/stats
> #---------------------------------------------------------------------
> # common defaults that all the 'listen' and 'backend' sections will
> # use if not designated in their block
> #---------------------------------------------------------------------  
> defaults
>     mode                    http
>     log                     global
>     option                  httplog
>     option                  dontlognull
>     option http-server-close
>     option forwardfor       except 127.0.0.0/8
>     option                  redispatch
>     retries                 3
>     timeout http-request    10s
>     timeout queue           1m
>     timeout connect         10s
>     timeout client          1m
>     timeout server          1m
>     timeout http-keep-alive 10s
>     timeout check           10s
>     maxconn                 3000
> #---------------------------------------------------------------------
> # kubernetes apiserver frontend which proxys to the backends
> #--------------------------------------------------------------------- 
> frontend kubernetes-apiserver
>     mode                 tcp
>     bind                 *:16443
>     option               tcplog
>     default_backend      kubernetes-apiserver    
> #---------------------------------------------------------------------
> # round robin balancing between the various backends
> #---------------------------------------------------------------------
> backend kubernetes-apiserver
>     mode        tcp
>     balance     roundrobin                                    # 负载策略
>     server      master01.k8s.io   10.10.10.11:6443 check      # master1节点地址
>     server      master02.k8s.io   10.10.10.12:6443 check      # master2节点地址
> #---------------------------------------------------------------------
> # collection haproxy statistics message
> #---------------------------------------------------------------------
> listen stats
>     bind                 *:1080
>     stats auth           admin:awesomePassword
>     stats refresh        5s
>     stats realm          HAProxy\ Statistics
>     stats uri            /admin?stats
> EOF

### --- 两台master都启动
~~~     设置开机启动

[root@k8s-master1 ~]# systemctl enable haproxy
[root@k8s-master2 ~]# systemctl enable haproxy

~~~     开启haproxy

[root@k8s-master1 ~]# systemctl start haproxy
[root@k8s-master2 ~]# systemctl start haproxy

~~~     查看启动状态

[root@k8s-master1 ~]# systemctl status haproxy
[root@k8s-master2 ~]# systemctl status haproxy

~~~     检查端口

[root@k8s-master1 ~]# netstat -lntup|grep haproxy
tcp        0      0 0.0.0.0:1080            0.0.0.0:*               LISTEN      11945/haproxy       
tcp        0      0 0.0.0.0:16443           0.0.0.0:*               LISTEN      11945/haproxy       
udp        0      0 0.0.0.0:34302           0.0.0.0:*                           11944/haproxy       
[root@k8s-master2 ~]# netstat -lntup|grep haproxy
tcp        0      0 0.0.0.0:1080            0.0.0.0:*               LISTEN      11945/haproxy       
tcp        0      0 0.0.0.0:16443           0.0.0.0:*               LISTEN      11945/haproxy       
udp        0      0 0.0.0.0:34302           0.0.0.0:*                           11944/haproxy