架构
Kubernetes高可用集群有两种部署方式,本文以第一种方式部署。
系统环境
cat /etc/redhat-release
CentOS Linux release 7.7.1908 (Core)
uname -a
Linux k8s-master 3.10.0-1062.9.1.el7.x86_64 #1 SMP Fri Dec 6 15:49:49 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
vi /etc/hosts
192.168.1.60 vip.k8s.twingao.com #虚拟IP地址,用于负载均衡,在三个主节点之间漂移。
192.168.1.61 k8s-master1 #主节点1 + HAProxy + Keepalived
192.168.1.62 k8s-master2 #主节点2 + HAProxy + Keepalived
192.168.1.63 k8s-master3 #主节点3 + HAProxy + Keepalived
192.168.1.64 k8s-node1 #工作节点1
192.168.1.65 k8s-node2 #工作节点2
192.168.1.66 k8s-node3 #工作节点3
关闭防火墙和安全设置。
systemctl stop firewalld
systemctl disable firewalld
vi /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0
vi /etc/selinux/config
SELINUX=disabled
#重启生效
reboot
安装Keepalived
yum install -y keepalived
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
vi /etc/keepalived/keepalived.conf
# k8s-maseter1
! Configuration File for keepalived
global_defs {
router_id k8s-master1 #主机名
}
vrrp_instance VI_1 {
state MASTER #不用修改,主要按照优先级确定
interface ens33 #vip所在的网卡
virtual_router_id 51
priority 150 #优先级,越大越优先
advert_int 1 #MASTER与BACKUP同步通知间隔,单位为秒
authentication { #同一vrrp实例的MASTER与BACKUP使用相同的密码才能正常通信
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.60 #vip
}
}
# k8s-maseter2
! Configuration File for keepalived
global_defs {
router_id k8s-master2
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.60
}
}
# k8s-maseter3
! Configuration File for keepalived
global_defs {
router_id k8s-master3
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 50
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.60
}
}
systemctl start keepalived
systemctl enable keepalived
由于k8s-master1节点的优先级最高,vip应该在k8s-master1,其它节点没有vip。
ip addr show ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:3d:77:8f brd ff:ff:ff:ff:ff:ff
inet 192.168.1.61/24 brd 192.168.1.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.1.60/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe3d:778f/64 scope link
valid_lft forever preferred_lft forever
安装HAProxy
yum install -y haproxy
cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak
vi /etc/haproxy/haproxy.cfg
#三个master节点的配置相同。
#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode tcp #改为tcp
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend main *:8443 #改为8443,为HAProxy监听客户端的端口。
acl url_static path_beg -i /static /images /javascript /stylesheets
acl url_static path_end -i .jpg .gif .png .css .js
use_backend static if url_static
default_backend k8s #改为k8s,与backend k8s对应
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
backend static
balance roundrobin
server static 127.0.0.1:4331 check
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend k8s #负载均衡k8s的api-server,对应三个k8s-master节点。
balance roundrobin
server k8s-master1 192.168.1.61:6443 check
server k8s-master2 192.168.1.62:6443 check
server k8s-master3 192.168.1.63:6443 check
systemctl start haproxy
systemctl enable haproxy
安装Kubernetes
修改内核参数。
cat <<EOF> /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
modprobe br_netfilter
sysctl --system
* Applying /usr/lib/sysctl.d/00-system.conf ...
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
* Applying /usr/lib/sysctl.d/10-default-yama-scope.conf ...
kernel.yama.ptrace_scope = 0
* Applying /usr/lib/sysctl.d/50-default.conf ...
kernel.sysrq = 16
kernel.core_uses_pid = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.all.promote_secondaries = 1
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
* Applying /etc/sysctl.d/99-sysctl.conf ...
* Applying /etc/sysctl.d/k8s.conf ...
#注意需要有以下两行
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
* Applying /etc/sysctl.conf ...
安装Docker。
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
或者
#yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum list docker-ce --showduplicates | sort -r
已加载插件:fastestmirror
可安装的软件包
* updates: mirrors.huaweicloud.com
Loading mirror speeds from cached hostfile
* extras: mirror.bit.edu.cn
docker-ce.x86_64 3:19.03.5-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.4-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.3-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.2-3.el7 docker-ce-stable
docker-ce.x86_64 3:19.03.1-3.el7 docker-ce-stable
docker-ce.x86_64