目录
2.5.7 部署kube-controller-manager
一、集群环境准备
1.1 主机规划
| 主机IP地址 | 主机名 | 主机配置 | 主机角色 | 软件列表 |
|---|---|---|---|---|
| 192.168.140.242 | k8s-master1 | 2C4G | master | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet、kube-proxy、Containerd、runc |
| 192.168.140.241 | k8s-master2 | 2C4G | master | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet、kube-proxy、Containerd、runc |
| 192.168.140.130 | k8s-worker1 | 2C4G | worker | kubelet、kube-proxy、Containerd、runc |
| 192.168.140.131 | ha1 | 1C2G | LB | haproxy、keepalived |
| 192.168.140.132 | ha2 | 1C2G | LB | haproxy、keepalived |
| 192.168.140.100 | / | / | VIP(虚拟IP) |
1.2 软件版本
| 软件名称 | 版本 | 备注 |
|---|---|---|
| CentOS7 | kernel版本：5.17 | |
| kubernetes | v1.25.4 | |
| etcd | v3.5.6 | |
| calico | v3.24.1 | |
| coredns | v1.9.3 | |
| containerd | 1.6.10 | |
| runc | 1.1.3 | 至少支持 v1alpha2 版本的容器运行时接口 |
| haproxy | 5.18 | YUM源默认 |
| keepalived | 3.5 | YUM源默认 |
1.3 网络分配
| 网络名称 | 网段 | 备注 |
|---|---|---|
| Node网络 | 192.168.140.0/24 | |
| Service网络 | 10.96.0.0/16 | |
| Pod网络 | 10.244.0.0/16 |
二、集群部署
2.1主机准备
2.1.1 主机名设置
hostnamectl set-hostname xxx
# 关于主机名参见1.1小节主机规划表2.1.2 主机与IP地址解析
cat >> /etc/hosts << EOF
192.168.140.131 ha1
192.168.140.132 ha2
192.168.140.242 k8s-master1
192.168.140.241 k8s-master2
192.168.140.130 k8s-worker1
EOF2.1.3 主机安全设置
2.1.3.1 关闭防火墙
systemctl stop firewalld
systmctl disable firewalld
firewall-cmd --state2.1.3.2 关闭selinux
setenforce 0
sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
sestatus2.1.4 交换分区设置
swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab
echo "vm.swappiness=0" >> /etc/sysctl.conf
sysctl -p2.1.5 主机系统时间同步
# 安装软件
yum -y install ntpdate
# 制定时间同步计划任务
crontab -e
0 */1 * * * ntpdate ntp1.aliyun.com2.1.6 主机系统优化
limit优化
ulimit -SHn 65535
cat <<EOF >> /etc/security/limits.conf
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF2.1.7 ipvs管理工具安装及模块加载
为集群节点安装，负载均衡节点不用安装
yum -y install ipvsadm ipset sysstat conntrack libseccomp# 所有节点配置ipvs模块，在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack， 4.18以下使用nf_conntrack_ipv4即可：
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack # 创建 /etc/modules-load.d/ipvs.conf 并加入以下内容：
cat >/etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF2.1.8 加载containerd相关内核模块
# 临时加载模块
modprobe overlay
modprobe br_netfilter
# 永久性加载模块
cat > /etc/modules-load.d/containerd.conf << EOF
overlay
br_netfilter
EOF
#设置为开机启动，如果报错查看内核版本是否过低，过低先升级内核
systemctl enable --now systemd-modules-load.service2.1.9 Linux内核升级
在所有节点中安装,需要重新操作系统更换内核。
[root@localhost ~]# yum -y install perl
[root@localhost ~]# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
[root@localhost ~]# yum -y install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm
[root@localhost ~]# yum --enablerepo="elrepo-kernel" -y install kernel-ml.x86_64
[root@localhost ~]# grub2-set-default 0
[root@localhost ~]# grub2-mkconfig -o /boot/grub2/grub.cfg2.1.10 Linux内核优化
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 131072
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOFsysctl --system
# 所有节点配置完内核后，重启服务器，保证重启后内核依旧加载
reboot -h now
# 重启后查看ipvs模块加载情况：
lsmod | grep --color=auto -e ip_vs -e nf_conntrack
# 重启后查看containerd相关模块加载情况：
lsmod | egrep 'br_netfilter | overlay'2.1.11 其它工具安装(选装)
yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git lrzsz -y2.2 负载均衡器准备
2.2.1 安装haproxy与keepalived
yum -y install haproxy keepalived2.2.2 HAProxy配置
cat >/etc/haproxy/haproxy.cfg<<"EOF"
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor
frontend k8s-master
bind 0.0.0.0:6443
bind 127.0.0.1:6443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-master1 192.168.140.242:6443 check
server k8s-master2 192.168.140.241:6443 check
EOF2.2.3 KeepAlived
主从配置不一致，需要注意。
ha1:
cat >/etc/keepalived/keepalived.conf<<"EOF"
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
script_user root
enable_script_security
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface ens33 # 跟据实际情况配置
mcast_src_ip 192.168.140.131 # 本机IP
virtual_router_id 51 # 虚拟路由ID
priority 100 # 优先级，主服务要设置比从服务高
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.140.100
}
track_script {
chk_apiserver
}
}
EOFha2:
cat >/etc/keepalived/keepalived.conf<<"EOF"
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
script_user root
enable_script_security
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
mcast_src_ip 192.168.140.132
virtual_router_id 51
priority 99
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.140.100
}
track_script {
chk_apiserver
}
}
EOF2.2.4 健康检查脚本
ha1及ha2均要配置
cat > /etc/keepalived/check_apiserver.sh <<"EOF"
#!/bin/bash
# 通过监控haproxy状态码，实现虚拟IP偏移
err=0
for k in $(seq 1 3)
do
check_code=$(pgrep haproxy)
if [[ $check_code == "" ]]; then
err=$(expr $err + 1)
sleep 1
continue
else
err=0
break
fi
done
if [[ $err != "0" ]]; then
echo "systemctl stop keepalived"
/usr/bin/systemctl stop keepalived
exit 1
else
exit 0
fi
EOFchmod +x /etc/keepalived/check_apiserver.sh2.2.5 启动服务并验证
systemctl daemon-reload
systemctl enable --now haproxy
systemctl enable --now keepalived
ip address show2.3 配置免密登录
在k8s-master1上操作
# 生成密钥
ssh-keygen
# 复制公钥到各主机
ssh-copy-id -p 2223 root@k8s-master1
ssh-copy-id -p 2223 root@k8s-master2
ssh-copy-id -p 2223 root@k8s-master3
ssh-copy-id -p 2223 root@k8s-worker1
ssh root@k8s-master12.4 部署ETCD集群
在k8s-master1上操作。
2.4.1 创建工作目录
mkdir -p /data/k8s-work2.4.2 获取cfssl工具
root@master:~# cd /data/k8s-work
root@master:~# wget https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64
root@master:~# wget https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64
root@master:~# wget https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl-certinfo_1.5.0_linux_amd64
# 说明：
# cfssl是使用go编写，由CloudFlare开源的一款PKI/TLS工具。主要程序有：
# - cfssl，是CFSSL的命令行工具
# - cfssljson用来从cfssl程序获取JSON输出，并将证书，密钥，CSR和bundle写入文件中。会生成证书的请求文件、证书的密钥和签发以后的证书
# - cfssl-certinfo是证书相关信息的查看工具
root@master:~#chmod +x cfssl*
root@master:~# mv cfssl_linux-amd64 /usr/local/bin/cfssl
root@master:~# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
root@master:~# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
root@master:~# cfssl version
Version: 1.2.0
Revision: dev
Runtime: go1.62.4.3 创建CA证书
2.4.3.1 配置ca证书请求文件
cat > ca-csr.json <<"EOF"
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "kubemsb",
"OU": "CN"
}
],
"ca": {
"expiry": "87600h"
}
}
EOF
# CN- 通用名称
# algo- 算法、size- 大小
# C- 国家、ST- 省份、L- 城市、O- 公司、OU- 部门
# expiry- 有效期2.4.3.2 创建ca证书
root@master:~# cfssl gencert -initca ca-csr.json | cfssljson -bare ca2.4.3.3 配置ca证书策略
root@master:~# cfssl print-defaults config > ca-config.jsoncat > ca-config.json <<"EOF"
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
# server auth 表示client可以对使用该ca对server提供的证书进行验证
# client auth 表示server可以使用该ca对client提供的证书进行验证2.4.4 创建etcd证书
2.4.4.1 配置etcd请求文件
最低0.47元/天 解锁文章
1247
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
