[root@k8s-master01 lb]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.229.199 k8s-vip
192.168.229.200 k8s-master01
192.168.229.201 k8s-master02
192.168.229.202 k8s-master03
192.168.229.203 k8s-slave01
192.168.229.204 k8s-slave02
[root@k8s-master01 lb]#
[root@localhost ~]#
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.229.200
PREFIX=24
GATEWAY=192.168.229.2
DNS1=192.168.229.2
[root@localhost ~]# yum install epel-release -y
[root@localhost ~]# yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
[root@localhost ~]# hostnamectl set-hostname k8s-master01
设置防火墙为iptables 并设置空规则
systemctl stop firewalld && systemctl disable firewalld
yum install -y iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save
关闭SELINUX
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
调整内核参数,对于k8s
[root@harbor ~]# modprobe br_netfilter
[root@k8s-master01 ~]#
cat > kubernetes.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间,只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
调整系统时区
timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 0
systemctl restart rsyslog
systemctl restart crond
关闭不需要的服务
systemctl stop postfix && systemctl disable postfix
设置rsyslogd和systemd journald
mkdir /var/log/journal
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf << EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent
# 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000
# 最大占用空间 10G
SystemMaxUse=10G
# 单日志文件最大 200M
SystemMaxFileSize=200M
# 日志保存时间 2 周
MaxRetentionSec=2week
# 不将日志转发到 syslog
ForwardToSyslog=no
EOF
systemctl restart systemd-journald
升级系统内核为 4.44
查看 Linux 所有内核命令
awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg)
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum --enablerepo=elrepo-kernel install -y kernel-lt
grub2-set-default "CentOS Linux (4.4.198-1.el7.elrepo.x86_64) 7 (Core)"
reboot # 设置完成后重启
# 重启后检测内核版本
[root@k8s-master01 ~]# uname -r
4.4.223-1.el7.elrepo.x86_64
[root@k8s-master01 ~]#
关闭NUMA
cp /etc/default/grub{,.bak}
vim /etc/default/grub # 在 GRUB_CMDLINE_LINUX 一行添加 numa=off 参数,如下所示:
[root@k8s-master01 ~]# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet numa=off"
GRUB_DISABLE_RECOVERY="true"
[root@k8s-master01 ~]#
生成默认文件的备份
cp /boot/grub2/grub.cfg{,.bak}
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot
Kubeadm 部署安装
kube-proxy 开启 ipvs 的前置条件
modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
安装docker软件
# 依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
reboot
# 启动 Docker 并设为开机自启
systemctl start docker
systemctl enable docker
cat > /etc/docker/daemon.json << EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"registry-mirrors": ["https://p02s6s7i.mirror.aliyuncs.com"]
}
EOF
mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload && systemctl restart docker && systemctl enable docker
在主节点启动Haproxy与Keepalived容器
安装Kubeadmin(主从配置)
cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum -y install kubeadm-1.15.1 kubectl-1.15.1 kubelet-1.15.1
systemctl enable kubelet.service
预先拉取镜像
由于 k8s.gcr.io 无法在国内访问到,所以预先加载一下构建集群所需的镜像。
# 在 master01 节点上执行,其他节点从 master01 节点复制
# 创建镜像列表文件 images
登载镜像
[root@k8s-master01 kubernetes_3_master_ha]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
quay.io/coreos/flannel v0.12.0-amd64 4e9f801d2217 2 months ago 52.8MB
grafana/grafana 6.6.0 199e063bb422 3 months ago 233MB
gcr.io/kubernetes-helm/tiller v2.14.3 2d0a693df3ba 9 months ago 94.2MB
k8s.gcr.io/kube-apiserver v1.15.1 68c3eb07bfc3 10 months ago 207MB
k8s.gcr.io/kube-scheduler v1.15.1 b0b3c4c404da 10 months ago 81.1MB
k8s.gcr.io/kube-controller-manager v1.15.1 d75082f1d121 10 months ago 159MB
k8s.gcr.io/kube-proxy v1.15.1 89a062da739d 10 months ago 82.4MB
quay.io/prometheus/node-exporter v0.18.1 e5a616e4b9cf 11 months ago 22.9MB
quay.io/coreos/flannel v0.11.0-amd64 ff281650a721 15 months ago 52.6MB
quay.io/coreos/kube-rbac-proxy v0.4.1 70eeaa7791f2 15 months ago 41.3MB
k8s.gcr.io/coredns 1.3.1 eb516548c180 16 months ago 40.3MB
k8s.gcr.io/kubernetes-dashboard-amd64 v1.10.1 f9aed6605b81 17 months ago 122MB
k8s.gcr.io/etcd 3.3.10 2c4adeb21b4f 17 months ago 258MB
wise2c/keepalived-k8s latest 0ba6a7862982 23 months ago 14MB
wise2c/haproxy-k8s latest fde31577093d 24 months ago 71.1MB
k8s.gcr.io/pause 3.1 da86e6ba6ca1 2 years ago 742kB
[root@k8s-master01 kubernetes_3_master_ha]# cat load_image.sh
FILE="
coredns.tar
dashboard.tar
etcd.tar
flannel1.tar
flannel.tar
grafana.tar
haproxy-k8s.tar
keepalived.tar
kube-apiserver.tar
kube-controller-manager.tar
kube-proxy.tar
pause.tar
prometheus.tar
rbac.tar
scheduler.tar
tiller.tar
"
for i in $FILE
do
docker load < $i
done
[root@k8s-master01 kubernetes_3_master_ha]# ./load_image.sh
到此结束,关闭虚拟机,做虚拟机的克隆
[root@k8s-master01 kubernetes_3_master_ha]# cp -r data /data
重启虚拟机即可
mkdir -p /etc/systemd/system/docker.service.d
systemctl daemon-reload && systemctl restart docker && systemctl enable docker
mkdir /usr/local/kubernetes
cd !$
cd /usr/local/kubernetes
[root@k8s-master01 kubernetes]# cd /usr/local/kubernetes
[root@k8s-master01 kubernetes]# cd /data/lb
[root@k8s-master01 lb]# ls
etc kubeadm-config.yaml start-haproxy.sh start-keepalived.sh
[root@k8s-master01 lb]# ls etc/
haproxy.cfg
[root@k8s-master01 lb]#
[root@k8s-master01 lb]# tail -f etc/haproxy.cfg
log global
balance roundrobin
server rancher01 192.168.229.200:6443
[root@k8s-master01 lb]# cat start-haproxy.sh
#!/bin/bash
MasterIP1=192.168.229.200
MasterIP2=192.168.229.201
MasterIP3=192.168.229.202
MasterPort=6443
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
-e MasterIP1=$MasterIP1 \
-e MasterIP2=$MasterIP2 \
-e MasterIP3=$MasterIP3 \
-e MasterPort=$MasterPort \
-v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
wise2c/haproxy-k8s
[root@k8s-master01 lb]#
[root@k8s-master01 lb]# chmod 777 start-haproxy.sh
[root@k8s-master01 lb]# ./start-haproxy.sh
[root@k8s-master01 lb]# netstat -anpt | grep :6444
tcp6 0 0 :::6444 :::* LISTEN 2003/docker-proxy
[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.229.199
INTERFACE=ens33
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s
[root@k8s-master01 lb]# chmod 777 ./start-keepalived.sh
[root@k8s-master01 lb]# ./start-keepalived.sh
[root@k8s-master01 lb]# ip addr show | grep 192.168.229
inet 192.168.229.200/24 brd 192.168.229.255 scope global noprefixroute ens33
inet 192.168.229.199/24 scope global secondary ens33
[root@k8s-master01 lb]#
[root@k8s-master01 lb]# cat /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
[root@k8s-master01 lb]#
yum install -y kubeadm-1.15.1 kubectl-1.15.1 kubelet-1.15.1
systemctl enable kubelet.service
[root@k8s-master01 ~]# kubeadm config print init-defaults > kubeadm-config.yaml
[root@k8s-master01 ~]# vi kubeadm-config.yaml
12 advertiseAddress: 192.168.229.200
26 controlPlaneEndpoint: "192.168.229.199:6444"
35 kubernetesVersion: v1.15.1
36 networking:
37 dnsDomain: cluster.local
38 podSubnet: "10.244.0.0/16"
39 serviceSubnet: 10.96.0.0/12
40 scheduler: {}
41 ---
42 apiVersion: kubeproxy.config.k8s.io/v1alpha1
43 kind: KubeProxyConfiguration
44 featureGates:
45 SupportIPVSProxyMode: true
46 mode: ipvs
[root@k8s-master01 install]#
[root@k8s-master01 install]# vi kubeadm-config.yaml
[root@k8s-master01 ~]# kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:5101d70fcb3c84345801e892fec6deed60b7cac83be2cc358dac4a9de482c6c3 \
--control-plane --certificate-key 43220eb7d1b8de372f8ec308dc1601ebdb527cf0ce6baf0b38dded519e2ab9fc
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:5101d70fcb3c84345801e892fec6deed60b7cac83be2cc358dac4a9de482c6c3
[root@k8s-master01 ~]#
[root@k8s-master01 ~]# vi .kube/config
现在先保留一个192.168.229.200,不用动,否则会出现启动一个未启动的节点上
[root@k8s-master01 ~]# scp -r /data root@k8s-master02:/
[root@k8s-master01 ~]# scp -r /data root@k8s-master03:/
[root@k8s-master02 ~]# cd /data/lb/
[root@k8s-master02 lb]# ls
etc kubeadm-config.yaml start-haproxy.sh start-keepalived.sh
[root@k8s-master02 lb]# ./start-haproxy.sh
[root@k8s-master02 lb]# ./start-keepalived.sh
[root@k8s-master02 lb]#
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:6ecf25b8332b493b271a1bd5fca19c95f8fd1a12f65c3ad9664685b61722f396 \
--control-plane --certificate-key e0df04746ea328085164b18c530547243fd371859e6287c308d2ce9a8aac7481
[root@k8s-master02 ~]# mkdir -p $HOME/.kube
[root@k8s-master02 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master02 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@k8s-master02 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 NotReady master 16m v1.15.1
k8s-master02 NotReady master 2m17s v1.15.1
[root@k8s-master02 ~]#
[root@k8s-master02 .kube]# cat config | grep server
server: https://192.168.229.199:6444
[root@k8s-master02 .kube]#
[root@k8s-master03 lb]# ./start-haproxy.sh
[root@k8s-master03 lb]# systemctl enable kubelet.service
[root@k8s-master03 lb]# kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:6ecf25b8332b493b271a1bd5fca19c95f8fd1a12f65c3ad9664685b61722f396 \
--control-plane --certificate-key e0df04746ea328085164b18c530547243fd371859e6287c308d2ce9a8aac7481
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@k8s-master01 ~]# docker ps| grep HAProxy-K8S
1fe211bad592 wise2c/haproxy-k8s "/docker-entrypoint.…" 3 hours ago Up 3 hours 0.0.0.0:6444->6444/tcp HAProxy-K8S
[root@k8s-master01 ~]#
[root@k8s-master01 ~]# cd /data/lb/
[root@k8s-master01 lb]# vi etc/haproxy.cfg
[root@k8s-master01 lb]# cat etc/haproxy.cfg
server rancher01 192.168.229.200:6443
server rancher02 192.168.229.201:6443
server rancher03 192.168.229.202:6443
[root@k8s-master01 lb]# docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh
[root@k8s-master01 lb]# scp etc/haproxy.cfg root@k8s-master02:/data/lb/etc/haproxy.cfg
[root@k8s-master01 lb]# scp etc/haproxy.cfg root@k8s-master03:/data/lb/etc/haproxy.cfg
[root@k8s-master02 ~]# docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh
[root@k8s-master03 ~]# docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh
安装flannel
必须先创建RBAC 否侧会失败
https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
下载地址kube-flannel.yml文件
https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml
[root@k8s-master01 ~]# kubectl apply -f kube-flannel.yml
到此还是不能够实现HA,,改成本机IP和6443,不是6444
[root@k8s-master02 ~]# cat .kube/config | grep server
server: https://192.168.229.201:6443
[root@k8s-master03 ~]# cat .kube/config | grep server
server: https://192.168.229.202:6443
[root@k8s-master01 ~]# cat .kube/config | grep server
server: https://192.168.229.200:6443
[root@k8s-master01 ~]#
查看集群状态
查看kube-controller-manager工作在哪个节点
[root@k8s-master01 ~]# kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-master01_92c8d2d4-4d75-4460-ac0e-a13b78006f5c","leaseDurationSeconds":15,"acquireTime":"2020-05-20T05:38:22Z","renewTime":"2020-05-20T05:52:28Z","leaderTransitions":1}'
creationTimestamp: "2020-05-20T05:36:31Z"
name: kube-controller-manager
namespace: kube-system
resourceVersion: "2103"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
uid: 6252f1e8-0b47-4391-a894-d21b6370317c
[root@k8s-master01 ~]# 查看kube-scheduler运行节点
[root@k8s-master01 ~]# kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml
apiVersion: v1
kind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-master02_f86541e5-e83a-4845-98d1-7abe25b9af16","leaseDurationSeconds":15,"acquireTime":"2020-05-20T05:38:23Z","renewTime":"2020-05-20T05:52:56Z","leaderTransitions":1}'
creationTimestamp: "2020-05-20T05:36:31Z"
name: kube-scheduler
namespace: kube-system
resourceVersion: "2152"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
uid: ee36393a-b022-4d72-864b-17489bd32127
[root@k8s-master01 ~]#
kube-api查看不了, 因为是一个HA
[root@k8s-master01 ~]# kubectl -n kube-system exec etcd-k8s-master01 -- etcdctl --endpoints=https://192.168.229.200:2379 --ca-file=/etc/kubernetes/pki/etcd/ca.crt --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key cluster-health
member 3cef022d23e1fa8c is healthy: got healthy result from https://192.168.229.202:2379
member 90e6e89202e61ff6 is healthy: got healthy result from https://192.168.229.201:2379
member 9b94aa8de130cb6d is healthy: got healthy result from https://192.168.229.200:2379
cluster is healthy
[root@k8s-master01 ~]#
安装Slave节点
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:5101d70fcb3c84345801e892fec6deed60b7cac83be2cc358dac4a9de482c6c3 \
--control-plane --certificate-key 43220eb7d1b8de372f8ec308dc1601ebdb527cf0ce6baf0b38dded519e2ab9fc
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:5101d70fcb3c84345801e892fec6deed60b7cac83be2cc358dac4a9de482c6c3
根据安装日志,这个输出,在安装slave节点的时候,只要执行
kubeadm join 192.168.229.199:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:5101d70fcb3c84345801e892fec6deed60b7cac83be2cc358dac4a9de482c6c3
就可以了
附录
[root@k8s-master01 lb]# tree
.
├── etc
│ └── haproxy.cfg
├── kubeadm-config.yaml
├── start-haproxy.sh
└── start-keepalived.sh
1 directory, 4 files
[root@k8s-master01 lb]# cat etc/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 192.168.229.200:6443
server rancher02 192.168.229.201:6443
server rancher03 192.168.229.202:6443
[root@k8s-master01 lb]# ls
etc kubeadm-config.yaml start-haproxy.sh start-keepalived.sh
[root@k8s-master01 lb]# cat start-haproxy.sh
#!/bin/bash
MasterIP1=192.168.229.200
MasterIP2=192.168.229.201
MasterIP3=192.168.229.202
MasterPort=6443
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
-e MasterIP1=$MasterIP1 \
-e MasterIP2=$MasterIP2 \
-e MasterIP3=$MasterIP3 \
-e MasterPort=$MasterPort \
-v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
wise2c/haproxy-k8s
[root@k8s-master01 lb]# cat start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.229.199
INTERFACE=ens33
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s
[root@k8s-master01 lb]# cd
[root@k8s-master01 ~]# ls
anaconda-ks.cfg haproxy.cfg kubeadm-config.yaml kubeadm-init.log kube-flannel.yml kubernetes_3_master_ha kubernetes.conf
[root@k8s-master01 ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.229.200
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8s-master01
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.229.199:6444"
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.15.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
[root@k8s-master01 ~]#