k8s二进制安装

kevin·zhao &&

已于 2023-06-08 22:17:23 修改

阅读量102

点赞数

文章标签： kubernetes linux 容器

于 2023-06-08 22:08:49 首次发布

本文链接：https://blog.csdn.net/zxcsd11/article/details/131116986

版权

该博客详细介绍了如何在Linux环境下手动安装k8s集群，包括集群结构、目录配置、环境准备、组件安装、证书生成、高可用设置、节点配置、网络插件(Calico)、DNS服务(CoreDNS)、监控(Metrics)和仪表板(Dashboard)的安装，以及集群验证和故障排查的步骤。

摘要由CSDN通过智能技术生成

在这里插入图片描述

集群结构

#master节点
192.168.0.107 master01
192.168.0.108 master02
192.168.0.109 master03

#work节点
192.168.0.110 node01
192.168.0.111 node02

#VIP，如果不是高可用集群，该IP为master01的IP
192.168.0.236

#K8s Service网段
10.96.0.0/12

#K8s Pod网段
172.16.0.0/12

#系统版本
CentOS Linux release 7.9.2009 (Core)

#内核版本
Linux k8s-master02 4.19.12-1.el7.elrepo.x86_64

目录结构

/etc/kubernetes/

环境配置

hosts文件

192.168.0.107 master01
192.168.0.108 master02
192.168.0.109 master03
192.168.0.236 master-lb
192.168.0.110 node01
192.168.0.111 node02

yum源

#下载yum源
cd /etc/yum.repos.d/
rm -rf *
wget http://mirrors.aliyun.com/repo/Centos-7.repo
wget http://mirrors.aliyun.com/repo/epel-7.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/Centos-7.repo

#重新设置缓存
yum clean all && yum makecache

#下载工具
yum -y install wget vim git telnet jq psmisc net-tools yum-utils device-mapper-persistent-data lvm2

#配置docker yum源
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

同步时间

#安装ntpdate
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum -y install ntpdate 

#配置同步时间
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com

#加入crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com

配置limit

vim /etc/security/limits.conf
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

关闭firewalld 、NetworkManager

systemctl disable --now firewalld 
systemctl disable --now NetworkManager

关闭selinux

#查看是否开启了SELinux
getenforce

#临时关闭
setenforce 0

#永久修改
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

关闭swap分区

#/etc/fstab注释swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

免密登录

#master01节点免密钥登录其他节点，安装过程中生成配置文件和证书均在master01上操作，集群管理也在master01上操作，阿里云或者AWS上需要单独一台kubectl服务器。
ssh-keygen -t rsa
for i in master02 master03 node01 node02;
do 
	ssh-copy-id -i /root/.ssh/id_rsa.pub $i;
done

升级系统

#CentOS7需要升级，CentOS8可以按需升级系统
yum -y update --exclude=kernel*

升级内核

#升级内核至4.18+，本地升级的版本为4.19,防止pod连不上，报错，宿主机宕机
cd /root
wget http://193.49.22.109/	elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm

#从master01节点传到其他节点：
for i in master02 master03;
do 
	scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm $i:/root/ ; 
done

#所有节点安装内核 	
cd /root
yum -y localinstall kernel-ml*
	
#所有节点更改内核启动顺序
grub2-set-default  0 
grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

#重启
reboot

#检查默认内核是不是4.19
grubby --default-kernel
#输出
/boot/vmlinuz-4.19.12-1.el7.elrepo.x86_64

#检查内核是不是4.19
uname -a
#输出
Linux k8s-master02 4.19.12-1.el7.elrepo.x86_64 #1 SMP Fri Dec 21 11:06:36 EST 2018 x86_64 x86_64 x86_64 GNU/Linux

ipvs

#所有节点安装ipvsadm：
yum -y install ipvsadm ipset sysstat conntrack libseccomp 

#所有节点配置ipvs模块/etc/modules-load.d/ipvs.conf，在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack，4.18以下使用nf_conntrack_ipv4即可
vim /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

#启动服务
systemctl enable --now systemd-modules-load.service

#检查是否加载
lsmod | grep -e ip_vs -e nf_conntrack
#输出
ip_vs_ftp              16384  0 
nf_nat                 32768  1 ip_vs_ftp
ip_vs_sed              16384  0 
ip_vs_nq               16384  0 
ip_vs_fo               16384  0 
ip_vs_sh               16384  0 
ip_vs_dh               16384  0 
ip_vs_lblcr            16384  0 
ip_vs_lblc             16384  0 
ip_vs_wrr              16384  0 
ip_vs_rr               16384  0 
ip_vs_wlc              16384  0 
ip_vs_lc               16384  0 
ip_vs                 151552  24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack          143360  2 nf_nat,ip_vs
nf_defrag_ipv6         20480  1 nf_conntrack
nf_defrag_ipv4         16384  1 nf_conntrack
libcrc32c              16384  4 nf_conntrack,nf_nat,xfs,ip_vs

开启集群k8s内核参数

cat  > /etc/sysctl.d/k8s.conf <<EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

#生效
sysctl --system

#所有节点配置完内核后，重启服务器，保证重启后内核依旧加载
reboot

安装组件

安装docker

#所有节点安装Docker-ce19.03
yum -y install docker-ce-19.03.* 

#由于新版kubelet建议使用systemd，所以可以把docker的CgroupDriver改成systemd
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

#开启Docker并设置开机自启动
systemctl daemon-reload && systemctl enable --now docker

#检查docker配置
docker info | grep Driver
#输出
 Storage Driver: overlay2
 Logging Driver: json-file
 Cgroup Driver: systemd

安装etcd

#下载etcd安装包
wget https://github.com/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz

#解压
tar -zxvf etcd-v3.4.13-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.4.13-linux-amd64/etcd{
   ,ctl}

#查看版本
etcdctl version

安装k8s

#下载
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG

#安装路径
/usr/local/bin/kube{
   let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

#解压安装
tar -zxvf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{
   let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

#查看版本
kubelet --version

发送etcd、k8s到其他节点

MasterNodes='master02 master03'
WorkNodes='node01 node02'
#主节点
for NODE in $MasterNodes; 
do 
	echo $NODE; 
	scp /usr/local/bin/kube{
   let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; 
	scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; 
done 
#工作节点
for NODE in $WorkNodes; 
do     
	scp /usr/local/bin/kube{
   let,-proxy} $NODE:/usr/local/bin/ ; 
done

安装文件

#创建/opt/cni/bin目录
mkdir -p /opt/cni/bin

#master01下载生成证书、基本组件等安装文件
cd /root
git clone https://github.com/dotbalo/k8s-ha-install.git

#安装文件切换分支
cd /root/k8s-ha-install
#查看分支
git branch -a
#输出
* master
  remotes/origin/HEAD -> origin/master
  remotes/origin/manual-installation
  remotes/origin/manual-installation-v1.16.x
  remotes/origin/manual-installation-v1.17.x
  remotes/origin/manual-installation-v1.18.x
  remotes/origin/manual-installation-v1.19.x
  remotes/origin/manual-installation-v1.20.x
  remotes/origin/manual-installation-v1.20.x-csi-hostpath
  remotes/origin/master
#切换到1.20.x分支
git checkout manual-installation-v1.20.x

etcd证书

安装工具

#CFSSL包含一个命令行工具和一个用于签名，验证和捆绑TLS证书的HTTP API工具

#master01下载生成证书工具
wget -O /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget -O /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 

#增加执行权限
chmod +x /usr/local/bin/cfssl 
chmod +x /usr/local/bin/cfssljson

cfssl

#默认配置模板
cfssl print-defaults config > config.json

#默认csr请求模板
cfssl print-defaults csr > csr.json

#ca-config.json：可以定义多个 profiles，分别指定不同的过期时间、使用场景等参数；后续在签名证书时使用某个 profile；此实例只有一个kubernetes模板。
#signing：表示该证书可用于签名其它证书；生成的 ca.pem 证书中 CA=TRUE；
#server auth：表示client可以用该 CA 对server提供的证书进行验证；
#client auth：表示server可以用该CA对client提供的证书进行验证；注意标点符号，最后一个字段一般是没有都好的。
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

#创建请求证书
#"CN"：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)
#"O"：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)
cat > ca-csr.json <<EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

master节点生成etcd证书

#master01节点创建etcd证书目录
mkdir -p /etc/etcd/ssl 

#生成证书的CSR文件：证书签名请求文件，配置域名、公司、单位(证书原理  公钥 私钥)
#生成etcd ca证书和ca证书的key
cd /root/k8s-ha-install/pki
cfssl gencert \
	-initca etcd-ca-csr.json | cfssljson \
	-bare /etc/etcd/ssl/etcd-ca
#输出
2021/04/05 18:14:26 [INFO] generating a new CA key and certificate from CSR
2021/04/05 18:14:26 [INFO] generate received request
2021/04/05 18:14:26 [INFO] received CSR
2021/04/05 18:14:26 [INFO] generating key: rsa-2048
2021/04/05 18:14:26 [INFO] encoded CSR
2021/04/05 18:14:26 [INFO] signed certificate with serial number 412473371421533418369334251703332009328607963759

#用etcd的ca证书颁发etcd的客户端证书
cfssl gencert \
	-ca=/etc/etcd/ssl/etcd-ca.pem \
	-ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
	-config=ca-config.json \
	-hostname=127.0.0.1,master01,master02,master03,192.168.0.107,192.168.0.108,192.168.0.109 \
	-profile=kubernetes etcd-csr.json | cfssljson 
	-bare /etc/etcd/ssl/etcd
#输出
2021/03/28 16:25:13 [INFO] generate received request
2021/03/28 16:25:13 [INFO] received CSR
2021/03/28 16:25:13 [INFO] generating key: rsa-2048
2021/03/28 16:25:13 [INFO] encoded CSR
2021/03/28 16:25:13 [INFO] signed certificate with serial number 456951054614724417062321915302964486528346522769

#复制master01的etcd证书到其他master节点
MasterNodes='192.168.0.108 192.168.0.109'
for NODE in $MasterNodes; do
     ssh $NODE "mkdir -p /etc/etcd/ssl"
     for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem; do
       scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}
     done
done

k8s证书

bukernetes证书

#所有节点创建kubernetes证书目录
mkdir -p /etc/kubernetes/pki  

#master01生成k8s ca证书和ca证书的key
cd /root/k8s-ha-install/pki
cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
#输出
2021/03/28 17:09:53 [INFO] generating a new CA key and certificate from CSR
2021/03/28 17:09:53 [INFO] generate received request
2021/03/28 17:09:53 [INFO] received CSR
2021