掌握核心技艺：Kubernetes安装前，宿主机配置参数

转发：掌握核心技艺：Kubernetes安装前，宿主机配置参数

 kubernetes生产级别安装：Kubernetes生产级别安装

1.背景

解决了用户在安装kubernetes时,配置内核参数及时间、limit的问题

2.基本环境配置

文档，https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/

2.1 关闭selinux等

所有节点关闭防火墙、selinux、dnsmasq、swap

systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager

setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

2.2 关闭swap

swapoff -a && sysctl -w vm.swappiness=0

sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

2.3 时间同步

#安装服务
yum install chrony -y

#启动服务
systemctl enable chronyd --now

#同步时间
[root@kube-master yum.repos.d]# chronyc sources
210 Number of sources = 4
MS Name/IP address         Stratum Poll Reach LastRx Last sample
===============================================================================
^- tock.ntp.infomaniak.ch        1  10   377   283    -10ms[  -10ms] +/-   87ms
^- ntp8.flashdance.cx            2  10   377   192    -18ms[  -18ms] +/-  154ms
^* 139.199.215.251               2  10   377   859   -902us[ -511us] +/-   59ms
^- tick.ntp.infomaniak.ch        1  10   377   27m    +26ms[  +26ms] +/-  133ms

• 或者

安装ntpdate

rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y

#操作如下
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com

2.4 limit配置

ulimit -SHn 65535

#添加配置文件
vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

2.5 内核配置

1.升级内核

CentOS7 需要升级内核至4.18+，本地升级的版本为4.19，所有节点升级

#下载
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm

#安装
yum localinstall -y kernel-ml*

#更改内核启动顺序
grub2-set-default  0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

#查看内核启动版本
[root@kube-master yum.repos.d]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64

#启动所有节点
reboot

#查看版本
uname -a

2.安装ipvsadm

yum install ipvsadm ipset sysstat conntrack libseccomp -y

• 配置ipvs模块

在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack

在内核4.18以下使用nf_conntrack_ipv4

vim /etc/modules-load.d/ipvs.conf 
# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

#启动
systemctl enable --now systemd-modules-load.service

• 或者其他方式

#启动时能自动加载模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

#授权
chmod 755 /etc/sysconfig/modules/ipvs.modules

#执行
sh /etc/sysconfig/modules/ipvs.modules

#查看
[root@kube-master yum.repos.d]# lsmod |grep -e ip_vs -e nf_conntrack
nf_conntrack_netlink    40960  0
nfnetlink              16384  3 nf_conntrack_netlink,ip_set
ip_vs_ftp              16384  0
nf_nat                 32768  2 nf_nat_ipv4,ip_vs_ftp
ip_vs_sed              16384  0
ip_vs_nq               16384  0
ip_vs_fo               16384  0
ip_vs_sh               16384  0
ip_vs_dh               16384  0
ip_vs_lblcr            16384  0
ip_vs_lblc             16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  0
ip_vs_wlc              16384  0
ip_vs_lc               16384  0
ip_vs                 151552  24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack          143360  6 xt_conntrack,nf_nat,ipt_MASQUERADE,nf_nat_ipv4,nf_conntrack_netlink,ip_vs
nf_defrag_ipv6         20480  1 nf_conntrack
nf_defrag_ipv4         16384  1 nf_conntrack
libcrc32c              16384  4 nf_conntrack,nf_nat,xfs,ip_vs

说明加载成功

3.配置必要参数

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
net.ipv4.conf.all.route_localnet = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

• 重新加载

sysctl --system

• 重启所有系统，查看模块是否加载好

reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack

2.6配置集群命令自动补全

https://kubernetes.io/zh/docs/tasks/tools/included/optional-kubectl-configs-bash-linux/

yum install bash-completion -y

echo 'source <(kubectl completion bash)' >>~/.bashrc

至此配置内核参数及时间、limit结束