第一章、Kubernetes基础之集群搭建（二进制安装）

qq_35302220

已于 2022-11-23 16:35:18 修改

阅读量908

点赞数

分类专栏： k8s系列文章标签： kubernetes docker 运维

于 2022-11-16 11:10:26 首次发布

本文链接：https://blog.csdn.net/qq_35302220/article/details/127846789

版权

本文档详细介绍了如何通过二进制方式搭建Kubernetes集群，包括环境配置、内核升级、组件安装、证书生成、系统组件配置、节点配置等关键步骤，确保集群的稳定和高效运行。

摘要由CSDN通过智能技术生成

集群安装

基本环境配置

10.103.236.201 k8s-master01 # 2C2G 40G
10.103.236.202 k8s-master02 # 2C2G 40G
10.103.236.203 k8s-master03 # 2C2G 40G
10.103.236.236 k8s-master-lb # VIP 虚IP不占用机器资源 # 如果不是高可用集群，该IP为Master01的IP
10.103.236.204 k8s-node01 # 2C2G 40G
10.103.236.205 k8s-node02  # 2C2G 40G

K8s Service网段：192.168.0.0/16
K8s Pod网段：172.16.0.0/12

系统环境：

[root@localhost ~]# cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)

配置所有节点hosts文件

[root@k8s-master01 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.103.236.201 k8s-master01
10.103.236.202 k8s-master02
10.103.236.203 k8s-master03
10.103.236.236 k8s-master-lb
10.103.236.204 k8s-node01
10.103.236.205 k8s-node02

CentOS 7安装yum源如下：

# centos7 yum源
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo

yum install -y yum-utils device-mapper-persistent-data lvm2

# doceker yum源
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

必备工具安装

yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

所有节点关闭firewalld 、dnsmasq、selinux(CentOS7需要关闭NetworkManager，CentOS8不需要)

systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager

setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

所有节点关闭swap分区，fstab注释swap

swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

所有节点同步时间

rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y

时间同步配置如下：

ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到 crontab -e 
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com

所有节点配置limit：

ulimit -SHn 65535

vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

Master01节点免密钥登录其他节点

安装过程中生成配置文件和证书均在Master01上操作，集群管理也在Master01上操作，阿里云或者AWS上需要单独一台kubectl服务器。

[root@k8s-master01 ~]# ssh-keygen -t rsa

Master01配置免密码登录其他节点

[root@k8s-master01 ~]# for i in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02;do ssh-copy-id -i .ssh/id_rsa.pub $i;done

所有节点安装基本工具

yum install wget jq psmisc vim net-tools yum-utils device-mapper-persistent-data lvm2 git -y

Master01下载安装文件

[root@k8s-master01 ~]# cd /root/ ; git clone https://github.com/dotbalo/k8s-ha-install.git
Cloning into 'k8s-ha-install'...
remote: Enumerating objects: 12, done.
remote: Counting objects: 100% (12/12), done.
remote: Compressing objects: 100% (11/11), done.
remote: Total 461 (delta 2), reused 5 (delta 1), pack-reused 449
Receiving objects: 100% (461/461), 19.52 MiB | 4.04 MiB/s, done.
Resolving deltas: 100% (163/163), done.

如果无法clone可以使用https://gitee.com/dukuan/k8s-ha-install.git进行克隆

所有节点升级系统并重启

此处升级没有升级内核，下节会单独升级内核：

yum update -y --exclude=kernel* && reboot     #CentOS7需要升级，CentOS8可以按需升级系统

内核升级

CentOS7 需要升级内核至4.18+，本地升级的版本为4.19

在master01节点下载内核：

cd /root
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm

从master01节点传到其他节点：

for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02;do scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm $i:/root/ ; done

所有节点安装内核

cd /root && yum localinstall -y kernel-ml*

所有节点更改内核启动顺序

grub2-set-default  0 && grub2-mkconfig -o /etc/grub2.cfg

grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

检查默认内核是不是4.19

[root@k8s-master01 ~]# grubby --default-kernel
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64

所有节点重启，然后检查内核是不是4.19

[root@k8s-master01 ~]# uname -a
Linux k8s-master01 4.19.12-1.el7.elrepo.x86_64 #1 SMP Fri Dec 21 11:06:36 EST 2018 x86_64 x86_64 x86_64 GNU/Linux

所有节点安装ipvsadm：

yum install ipvsadm ipset sysstat conntrack libseccomp -y

所有节点配置ipvs模块，在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack， 4.18以下使用nf_conntrack_ipv4即可：

modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack


vim /etc/modules-load.d/ipvs.conf 
# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

然后执行systemctl enable --now systemd-modules-load.service即可

检查是否加载：

[root@k8s-master01 ~]# lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_sh               16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  0
ip_vs                 151552  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          143360  1 ip_vs
nf_defrag_ipv6         20480  1 nf_conntrack
nf_defrag_ipv4         16384  1 nf_conntrack
libcrc32c              16384  3 nf_conntrack,xfs,ip_vs

开启一些k8s集群中必须的内核参数，所有节点配置k8s内核：

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
net.ipv4.conf.all.route_localnet = 1

vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system

所有节点配置完内核后，重启服务器，保证重启后内核依旧加载

reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack

基本组件安装

Docker安装

所有节点安装Docker-ce 20.10

yum install docker-ce-20.10.* docker-cli-20.10.* -y

温馨提示：
由于新版kubelet建议使用systemd，所以可以把docker的CgroupDriver改成systemd

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
   
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

所有节点设置开机自启动Docker：

systemctl daemon-reload && systemctl enable --now docker

K8s及etcd安装

Master01下载kubernetes安装包

[root@k8s-master01 ~]# wget https://dl.k8s.io/v1.21.0/kubernetes-server-linux-amd64.tar.gz

查看kubernetes版本：
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.21.md

以下操作都在master01执行

下载etcd安装包

[root@k8s-master01 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz

解压kubernetes安装文件

[root@k8s-master01 ~]# tar -xf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

解压etcd安装文件

[root@k8s-master01 ~]#  tar -zxvf etcd-v3.4.13-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.4.13-linux-amd64/etcd{,ctl}

版本查看

[root@k8s-master01 ~]# kubelet --version
Kubernetes v1.21.0

[root@k8s-master01 ~]# etcdctl version
etcdctl version: 3.4.13
API version: 3.4

将组件发送到其他节点

MasterNodes='k8s-master02 k8s-master03'
WorkNodes='k8s-node01 k8s-node02'
for NODE in $MasterNodes; do echo $NODE; scp /usr/local/bin/kube{
   let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done
for NODE in $WorkNodes; do     scp /usr/local/bin/kube{
   let,-proxy} $NODE:/usr/local/bin/ ; done

所有节点创建/opt/cni/bin目录

mkdir -p /opt/cni/bin

Master01节点切换到1.21.x分支（其他版本可以切换到其他分支）

cd k8s-ha-install && git checkout manual-installation-v1.21.x

生成证书

Master01下载生成证书工具

wget "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64" -O /usr/local/bin/cfssl
wget "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

etcd证书

所有Master节点创建etcd证书目录

mkdir /etc/etcd/ssl -p

所有节点创建kubernetes相关目录

mkdir -p /etc/kubernetes/pki

Master01节点生成etcd证书
生成证书的CSR文件：证书签名请求文件，配置了一些域名、公司、单位

[root@k8s-master01 pki]# cd /root/k8s-ha-install/pki

# 生成etcd CA证书和CA证书的key
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca


cfssl gencert \
   -ca=/etc/etcd/ssl/etcd-ca.pem \
   -ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
   -config=ca-config.json \
   -hostname=127.0.0.1,k8s-master01,k8s-master02,k8s-master03,10.103.236.201,10.103.236.202,10.103.236.203 \
   -profile=kubernetes \
   etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

#	执行结果
2022/11/15 10:05:25 [INFO] generate received request
2022/11/15 10:05:25 [INFO] received CSR
2022/11/15 10:05:25 [INFO] generating key: rsa-2048
2022/11/15 10:05:25 [INFO] encoded CSR
2022/11/15 10:05:25 [INFO] signed certificate with serial number 145091942380882075473679379022659360857185124151

将证书复制到其他节点

MasterNodes='k8s-master02 k8s-master03'
WorkNodes='k8s-node01 k8s-node02'

for NODE in $MasterNodes; do
     ssh $NODE "mkdir -p /etc/etcd/ssl"
     for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem; do
       scp /etc/etcd/ssl/${
   FILE} $NODE:/etc/etcd/ssl/${
   FILE}
     done
 done

k8s组件证书

Master01生成kubernetes证书

[root@k8s-master01 pki]# cd /root/k8s-ha-install/pki

[root@k8s-master01 pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca

# 192.168.0.是k8s service的网段，如果说需要更改k8s service网段，那就需要更改192.168.0.1，
# 如果不是高可用集群，10.103.236.236为Master01的IP
cfssl gencert   -ca=/etc/kubernetes/pki/ca.pem   -ca-key=/etc/kubernetes/pki/ca-key.pem   -config=ca-config.json   -hostname=192.168.0.1,10.103.236.236,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,10.103.236.201,10.103.236.202,10.103.236.203   -profile=kubernetes   apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

生成apiserver的聚合证书。Requestheader-client-xxx requestheader-allowwd-xxx:aggerator

cfssl gencert   -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca 

cfssl gencert   -ca=/etc/kubernetes/pki/front-proxy-ca.pem   -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem   -config=ca-config.json   -profile=kubernetes   front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

#	返回结果（忽略警告）
2022/11/15 10:45:00 [INFO] generate received request
2022/11/15 10:45:00 [INFO] received CSR
2022/11/15 10:45:00 [INFO] generating key: rsa-2048
2022/11/15 10:45:00 [INFO] encoded CSR
2022/11/15 10:45:00 [INFO] signed certificate with serial number 425494542797381273262084836805369391923874376347
2022/11/15 10:45:00 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requireme