二进制部署K8S高可用集群之基础环境准备

基础环境准备

基本信息

1. 修改IP地址

[root@harbor.oldboyedu.com ~]$ nmcli connection modify ens32 ipv4.addresses 10.0.0.201/24 && nmcli connection up ens32
[root@harbor.oldboyedu.com ~]$ nmcli connection modify ens32 ipv4.addresses 10.0.0.202/24 && nmcli connection up ens32
[root@harbor.oldboyedu.com ~]$ nmcli connection modify ens32 ipv4.addresses 10.0.0.203/24 && nmcli connection up ens32
[root@harbor.oldboyedu.com ~]$ nmcli connection modify ens32 ipv4.addresses 10.0.0.204/24 && nmcli connection up ens32
[root@harbor.oldboyedu.com ~]$ nmcli connection modify ens32 ipv4.addresses 10.0.0.205/24 && nmcli connection up ens32
[root@harbor.oldboyedu.com ~]$ 

2. 修改主机名

hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-master02
hostnamectl set-hostname k8s-master03
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02

3. 所有节点配置hosts文件解析

# 配置master01
[root@k8s-master01 ~]$ cat >> /etc/hosts <<'EOF'
> 10.0.0.201  k8s-master01
> 10.0.0.202  k8s-master02
> 10.0.0.203  k8s-master03
> 10.0.0.204  k8s-node01
> 10.0.0.205  k8s-node02
> EOF
[root@k8s-master01 ~]$ 
# 配置master02
# 配置master03
# 配置node01
# 配置node02

4. master01节点配置免密连接

[root@k8s-master01 ~]$ cat > password_free_login.sh <<'EOF'
#!/bin/bash
# auther: Jason Luo

# 创建密钥对
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa -q

# 声明你服务器密码，建议所有节点的密码均一致，否则该脚本需要再次进行优化
export mypasswd=119119

# 定义主机列表
k8s_host_list=(k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02)

# 配置免密登录，利用expect工具免交互输入
for i in ${k8s_host_list[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
  expect {
    \"*yes/no*\" {send \"yes\r\"; exp_continue}
    \"*password*\" {send \"$mypasswd\r\"; exp_continue}
  }"
done
EOF
[root@k8s-master01 ~]$
[root@k8s-master01 ~]$ sh password_free_login.sh

5. 所有节点基础软件安装

# 传输软件
[root@k8s-master01 ~]$ k8s_host_list=(k8s-master02 k8s-master03 k8s-node01 k8s-node02)
[root@k8s-master01 ~]$ echo ${k8s_host_list[@]}
k8s-master02 k8s-master03 k8s-node01 k8s-node02
[root@k8s-master01 ~]$ for i in ${k8s_host_list[@]};do scp -r commonly_used/ root@${i}:/root; done

# localinstall
# 网络安装 yum -y install bind-utils expect rsync wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git ntpdate
[root@k8s-master01 ~]$ yum -y localinstall commonly_used/*.rpm
[root@k8s-master02 ~]$ yum -y localinstall commonly_used/*.rpm
[root@k8s-master03 ~]$ yum -y localinstall commonly_used/*.rpm
[root@k8s-node01 ~]$ yum -y localinstall commonly_used/*.rpm
[root@k8s-node02 ~]$ yum -y localinstall commonly_used/*.rpm

6. master01节点配置自动同步

[root@k8s-master01 ~]$ cat > /usr/local/sbin/data_rsync.sh <<'EOF'
> #!/bin/bash
> # Auther: Jason Luo
>
> if  [ $# -ne 1 ];then
>    echo "Usage: $0 /path/to/file(绝对路径)"
>    exit
> fi
>
> if [ ! -e $1 ];then
>     echo "[ $1 ] dir or file not find!"
>     exit
> fi
>
> fullpath=`dirname $1`
>
> basename=`basename $1`
>
> cd $fullpath
>
> k8s_host_list=(k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02)
>
> for host in ${k8s_host_list[@]};do
>   tput setaf 2
>     echo ===== rsyncing ${host}: $basename =====
>     tput setaf 7
>     rsync -az $basename  `whoami`@${host}:$fullpath
>     if [ $? -eq 0 ];then
>       echo "命令执行成功!"
>     fi
> done
> EOF
[root@k8s-master01 ~]$ chmod +x /usr/local/sbin/data_rsync.sh
[root@k8s-master01 ~]$ cp /etc/hosts /tmp/
[root@k8s-master01 ~]$ data_rsync.sh /tmp/hosts
===== rsyncing k8s-master01: hosts =====
命令执行成功!
===== rsyncing k8s-master02: hosts =====
命令执行成功!
===== rsyncing k8s-master03: hosts =====
命令执行成功!
===== rsyncing k8s-node01: hosts =====
命令执行成功!
===== rsyncing k8s-node02: hosts =====
命令执行成功!
[root@k8s-master01 ~]$

Linux基础环境优化

1. 所有节点关闭firewalld，selinux，NetworkManager

systemctl disable --now firewalld
systemctl disable --now NetworkManager
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

2. 所有节点关闭swap分区，fstab注释swap

swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
free -h

3. 所有节点配置时间同步

*/2 * * * * /sbin/ntpdate ntp1.aliyun.com &> /dev/null

4. 所有节点配置limit

cat >> /etc/security/limits.conf <<'EOF'
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF

5. 所有节点优化sshd服务

[root@k8s-master01 ~]$ sed -i 's@#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
[root@k8s-master01 ~]$ sed -i 's@^GSSAPIAuthentication yes@GSSAPIAuthentication no@g' /etc/ssh/sshd_config
[root@k8s-master01 ~]$ grep UseDNS /etc/ssh/sshd_config
UseDNS no
[root@k8s-master01 ~]$ grep GSSAPIAuthentication /etc/ssh/sshd_config
GSSAPIAuthentication no
[root@k8s-master01 ~]$

6. 所有节点内核调优

[root@k8s-master01 ~]$ cat > /etc/sysctl.d/k8s.conf <<'EOF'
> net.ipv4.ip_forward = 1
> net.bridge.bridge-nf-call-iptables = 1
> net.bridge.bridge-nf-call-ip6tables = 1
> net.ipv6.conf.all.disable_ipv6 = 1
> fs.may_detach_mounts = 1
> vm.overcommit_memory=1
> vm.panic_on_oom=0
> fs.inotify.max_user_watches=89100
> fs.file-max=52706963
> fs.nr_open=52706963
> net.netfilter.nf_conntrack_max=2310720
> net.ipv4.tcp_keepalive_time = 600
> net.ipv4.tcp_keepalive_probes = 3
> net.ipv4.tcp_keepalive_intvl =15
> net.ipv4.tcp_max_tw_buckets = 36000
> net.ipv4.tcp_tw_reuse = 1
> net.ipv4.tcp_max_orphans = 327680
> net.ipv4.tcp_orphan_retries = 3
> net.ipv4.tcp_syncookies = 1
> net.ipv4.tcp_max_syn_backlog = 16384
> net.ipv4.ip_conntrack_max = 65536
> net.ipv4.tcp_max_syn_backlog = 16384
> net.ipv4.tcp_timestamps = 0
> net.core.somaxconn = 16384
> EOF
[root@k8s-master01 ~]$ sysctl --system
[root@k8s-master01 ~]$

7. 修改终端颜色 # 可选项

[root@k8s-master01 ~]# cat <<EOF >>  ~/.bashrc
> PS1='[\[\e[34;1m\]\u@\[\e[0m\]\[\e[32;1m\]\H\[\e[0m\]\[\e[31;1m\] \W\[\e[0m\]]# '
> EOF
[root@k8s-master01 ~]# source ~/.bashrc

8. 所有节点升级内核

[root@k8s-master01 kernel_update]# echo ${k8s_host_list[@]}
k8s-master02 k8s-master03 k8s-node01 k8s-node02
[root@k8s-master01 kernel_update]# for i in ${k8s_host_list[@]};do scp -r /root/kernel_update/ root@${i}:/root; done
[root@k8s-master01 ~]# yum localinstall -y /root/kernel_update/*.rpm

9. 所有节点修改内核启动顺序

[root@k8s-master01 ~]# grubby --default-kernel
/boot/vmlinuz-3.10.0-1160.el7.x86_64
[root@k8s-master01 ~]# grub2-set-default  0 && grub2-mkconfig -o /etc/grub2.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-4.19.12-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1160.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1160.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-e187278346824b6c8085c10ab378dffc
Found initrd image: /boot/initramfs-0-rescue-e187278346824b6c8085c10ab378dffc.img
done
[root@k8s-master01 ~]# grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
[root@k8s-master01 ~]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64
[root@k8s-master01 ~]#

10. 所有节点升级软件包 # 内核升级了，软件包也要对应地进行升级

[root@k8s-master01 ~]# for i in ${k8s_host_list[@]};do scp -r /root/update_software/ root@${i}:/root; done
[root@k8s-master01 ~]# yum localinstall update_software/*.rpm # 这一步耗时较长

所有节点安装ipvsadm以实现kube-proxy的负载均衡

[root@k8s-master01 ~]# mkdir ipvsadm
[root@k8s-master01 ~]# cd ipvsadm/
[root@k8s-master01 ipvsadm]# cd ..
[root@k8s-master01 ~]# for i in ${k8s_host_list[@]};do scp -r /root/ipvsadm/ root@${i}:/root; done
[root@k8s-master01 ~]# yum localinstall -y ipvsadm/*.rpm
[root@k8s-master01 ~]#

1. 所有节点手动加载模块

[root@k8s-master01 ~]# modprobe -- ip_vs
[root@k8s-master01 ~]# modprobe -- ip_vs_rr
[root@k8s-master01 ~]# modprobe -- ip_vs_wrr
[root@k8s-master01 ~]# modprobe -- ip_vs_sh
[root@k8s-master01 ~]# modprobe -- nf_conntrack

2. 所有节点创建要开机自动加载的模块配置文件

[root@k8s-master01 ~]# cat > /etc/modules-load.d/ipvs.conf << 'EOF'
> ip_vs
> ip_vs_lc
> ip_vs_wlc
> ip_vs_rr
> ip_vs_wrr
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_dh
> ip_vs_sh
> ip_vs_fo
> ip_vs_nq
> ip_vs_sed
> ip_vs_ftp
> ip_vs_sh
> nf_conntrack
> ip_tables
> ip_set
> xt_set
> ipt_set
> ipt_rpfilter
> ipt_REJECT
> ipip
> EOF
[root@k8s-master01 ~]#

3. 所有节点启动模块

[root@k8s-master01 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack
ip_vs_sh               12688  0
ip_vs_wrr              12697  0
ip_vs_rr               12600  0
ip_vs                 145458  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          139264  1 ip_vs
libcrc32c              12644  3 xfs,ip_vs,nf_conntrack
[root@k8s-master01 ~]#

重启所有节点并检查内核和模块是否配置成功

1. 所有节点查看现有内核版本

[root@k8s-master01 ~]# uname -r
4.19.12-1.el7.elrepo.x86_64
[root@k8s-master01 ~]#

2. 所有节点检查默认加载的内核版本

[root@k8s-master01 ~]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64
[root@k8s-master01 ~]#

3. 所有节点检查支持ipvs的内核模块是否加载成功

[root@k8s-master01 ~]# lsmod | grep --color=auto -e ip_vs -e nf_conntrack
[root@k8s-master01 ~]#

所有节点做快照