2. 安装篇-K8S安装-二进制

错误手册

0. Kubernetes-Error

集群规划

主机规划

服务器名称	主机名	主机IP地址	资源配置	节点用途
/	k8s-master-lb	192.169.1.200	/	keepalived虚拟IP
K8S-1	k8s-master-201	192.169.1.201	2核|2G	master节点
K8S-2	k8s-master-202	192.169.1.202	2核|2G	master节点
K8S-3	k8s-master-203	192.169.1.203	2核|2G	master节点
K8S-4	k8s-worker-204	192.169.1.204	2核|2G	worker节点
K8S-5	k8s-worker-205	192.169.1.205	2核|2G	worker节点

网段规划

网段用途	网段范围	备注
宿主机网段	192.169.1.200/32-192.169.1.205/32
K8S Server网段	10.96.0.0/16
K8S Pod网段	172.168.0.0/16

软件版本

软件	版本	备注
linux	CentOS Linux release 7.9.2009 (Core)
kernel	6.3.3-1.el7.elrepo.x86_64
kubernetes	v1.26.5
docker	v20.10.24
etcd	v3.5.9
coredns
calico
harbor
dashboard
helm
metallb
ingress-nginx
metrics-server
prometheus
grafana
istio

基础环境配置

配置hostname

涉及节点：所有节点，逐个操作

hostnamectl set-hostname k8s-master-201 && hostname
hostnamectl set-hostname k8s-master-202 && hostname
hostnamectl set-hostname k8s-master-203 && hostname
hostnamectl set-hostname k8s-worker-204 && hostname
hostnamectl set-hostname k8s-worker-205 && hostname

配置hosts文件

涉及节点：所有节点

cat >> /etc/hosts << 'EOF'
192.169.1.200 k8s-master-lb
192.169.1.201 k8s-master-201
192.169.1.202 k8s-master-202
192.169.1.203 k8s-master-203
192.169.1.204 k8s-worker-204
192.169.1.205 k8s-worker-205
EOF

ping k8s-master-201 -c 1
ping k8s-master-202 -c 1
ping k8s-master-203 -c 1
ping k8s-worker-204 -c 1
ping k8s-worker-205 -c 1

配置yum源

涉及节点：所有节点
安装yum源

cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo_`date +%F`
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo

安装基础工具

yum -y install yum-utils device-mapper-persistent-data lvm2 vim wget telnet net-tools git

配置docker源

涉及节点：所有节点

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

关闭firewalld

涉及节点：所有节点

systemctl disable --now firewalld

关闭dnsmasq

涉及节点：所有节点

systemctl disable --now dnsmasq

关闭selinux

涉及节点：所有节点

setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config 

关闭swap

涉及节点：所有节点

swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

设置时区

涉及节点：所有节点

timedatectl set-timezone Asia/Shanghai

设置limit

涉及节点：所有节点

cat >> /etc/security/limits.conf << EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655360
* hard nproc 655360
* soft memlock unlimited
* hard memlock unlimited
EOF

安装ntpdate

涉及节点：所有节点

安装ntpdate

yum -y install epel-release
yum -y install ntpdate

配置时间同步

手动同步时间

ntpdate time2.aliyun.com

定时同步时间

crontab -e
*/5 * * * * ntpdate time2.aliyun.com

开机同步时间

cat >> /etc/rc.d/rc.local << 'EOF'
ntpdate time2.aliyun.com
EOF
chmod +x /etc/rc.d/rc.local

免密登录

涉及节点：k8s-master-201
配置k8s-master-201节点可以免密登录其他节点，用于安装过程中生成的配置文件和证书均在k8s-master-201上操作，集群管理也在k8s-master-201上操作。

生成秘钥

ssh-keygen -t rsa

发送密钥到其他节点

yum -y install sshpass
for HOST_NAME in k8s-master-201 k8s-master-202 k8s-master-203 k8s-worker-204 k8s-worker-205
do
  sshpass -p "cmk521" ssh-copy-id -o "StrictHostKeyChecking no" -i .ssh/id_rsa.pub $HOST_NAME
done

内核配置

涉及节点：所有节点

CentOS7

centos7内核需要升级至4.18以上
升级内核

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm

yum --enablerepo=elrepo-kernel install kernel-ml-devel kernel-ml -y
yum --enablerepo=elrepo-kernel install kernel-lt-devel kernel-lt -y

修改默认启动内容

awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
grub2-set-default 0
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot

检查内核版本

uname -r

注意：
通过命令awk -F’ ‘$1=="menuentry " {print i++ " : " $2}’ /etc/grub2.cfg可以查看到可用内核，及内核的序号。
通过命令grub2-set-default 0，设置新内核为默认启动的内核。
通过命令grub2-mkconfig -o /boot/grub2/grub.cfg生成grub文件。
通过reboot启动服务器即可。

CentOS8

centos8按需升级
升级内核

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
yum install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm

yum --enablerepo=elrepo-kernel install kernel-ml -y
yum --enablerepo=elrepo-kernel install kernel-lt -y

修改默认启动内容

#查看当前默认启动内核
grubby --default-kernel

#以新版本内核启动
grub2-set-default 0

#指定某个内核启动
grubby --set-default /boot/vmlinuz-5.19.2-1.el8.elrepo.x86_64

#重启主机
reboot

检查内核版本

uname -r

参数优化

涉及节点：所有节点

系统调优参数

cat > /etc/sysctl.d/k8s.conf <<EOF
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
# 要求iptables不对bridge的数据进行处理
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
vm.overcommit_memory=1
# 开启OOM
vm.panic_on_oom=0
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.swappiness=0
# ipvs优化
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
EOF

sysctl --system

文件最大打开数

cat > /etc/security/limits.d/k8s.conf <<EOF
*       soft    nproc   1048576
*       hard    nproc   1048576
*       soft    nofile  1048576
*       hard    nofile  1048576
root    soft    nproc   1048576
root    hard    nproc   1048576
root    soft    nofile  1048576
root    hard    nofile  1048576
EOF

优化日志处理，减少磁盘IO

sed -ri 's/^\$ModLoad imjournal/#&/' /etc/rsyslog.conf
sed -ri 's/^\$IMJournalStateFile/#&/' /etc/rsyslog.conf

sed -ri 's/^#(DefaultLimitCORE)=/\1=100000/' /etc/systemd/system.conf
sed -ri 's/^#(DefaultLimitNOFILE)=/\1=100000/' /etc/systemd/system.conf

ssh 连接优化

sed -ri 's/^#(UseDNS )yes/\1no/' /etc/ssh/sshd_config

基本组件安装

安装docker

涉及节点：所有节点
docker依赖

yum install -y yum-utils
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

docker可用版本（可忽略执行）

yum list docker-ce --showduplicates
yum list containerd.io --showduplicate
yum list docker-ce-rootless-extras --showduplicate

docker安装

yum -y install docker-ce-20.10.24-3.el7 \
docker-ce-cli-20.10.24-3.el7 \
docker-ce-rootless-extras-20.10.24-3.el7 \
containerd.io

docker配置

mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": [
    "https://mciwm180.mirror.aliyuncs.com",
    "https://docker.mirrors.ustc.edu.cn/",
    "https://registry.docker-cn.com"
  ],
  "log-driver": "json-file",
  "log-opts": {
    "max-file": "10",
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

docker启动

systemctl enable --now docker
systemctl enable --now containerd

docker当前版本

docker version

安装etcd

涉及节点：k8s-master-201
etcd网站
https://github.com/etcd-io/etcd/releases
etcd下载

wget https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-amd64.tar.gz

etcd安装

tar -zxvf etcd-v3.5.9-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-v3.5.9-linux-amd64/etcd{,ctl}

etcd版本

etcdctl version

etcd发送到所以master节点

MASTERNODES="k8s-master-202 k8s-master-203"
for NODE in $MASTERNODES; do
  echo $NODE
  scp /usr/local/bin/etcd* $NODE:/usr/local/bin/
done

etcd配置

安装kubernetes

涉及节点：k8s-master-201
kubernetes网站
https://github.com/kubernetes/kubernetes/blob/release-1.26/CHANGELOG/CHANGELOG-1.26.md
kubernetes下载

wget https://dl.k8s.io/v1.26.5/kubernetes-server-linux-amd64.tar.gz

kubernetes安装

tar -zxvf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

kubernetes版本

kubelet --version

命令补全

yum -y install bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

kubernetes发送到master节点和worker节点

MASTERNODES="k8s-master-202 k8s-master-203"
WORKERNODES="k8s-worker-204 k8s-worker-205"
for NODE in $MASTERNODES; do
  echo $NODE
  scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/
done

for NODE in $WORKERNODES; do
  echo $NODE
  scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/
done

安装文件

初始化仓库

git config --global user.name "wangquan"
git config --global user.email "15515190288@163.com"
git init

下载安装文件
参考 dotbalo 的文档，文档地址：https://github.com/dotbalo/k8s-ha-install

git clone https://github.com/dotbalo/k8s-ha-install.git

切换分支

cd /root/k8s-ha-install/
git branch -a
git checkout manual-installation-v1.26.x

生成证书

涉及节点：k8s-master-201
安装证书生成工具

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl{,json}

生成etcd证书

mkdir -p /etc/etcd/ssl
cd /root/k8s-ha-install/pki
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca
cfssl gencert \
-ca=/etc/etcd/ssl/etcd-ca.pem \
-ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
-config=ca-config.json \
-hostname=127.0.0.1,k8s-master-201,k8s-master-202,k8s-master-203,192.169.1.201,192.169.1.202,192.169.1.203 \
-profile=kubernetes \
etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd

发送etcd证书到其他master节点

MASTERNODES="k8s-master-202 k8s-master-203"
for NODE in $MASTERNODES; do
	echo $NODE
	ssh $NODE "mkdir -p /etc/etcd/ssl"
	for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do
  	scp /etc/etcd/ssl/$FILE $NODE:/etc/etcd/ssl/$FILE
	done
done

生成kubernetes证书-apiserver

mkdir -p /etc/kubernetes/pki
cd /root/k8s-ha-install/pki

cd /root/k8s-ha-install/pki
#生成apiserver证书
cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-hostname=10.96.0.1,192.169.1.200,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.169.1.201,192.169.1.202,192.169.1.203 \
-profile=kubernetes \
apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

#生成apiserver聚合证书
cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca
cfssl gencert \
-ca=/etc/kubernetes/pki/front-proxy-ca.pem \
-ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

生成kubernetes证书-controller-manager

cd /root/k8s-ha-install/pki
#生成controller-manager证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager

#设置集群项
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.169.1.200:8443 \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

#设置环境项
kubectl config set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

#设置用户项
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=/etc/kubernetes/pki/controller-manager.pem \
--client-key=/etc/kubernetes/pki/controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

#设置默认环境
kubectl config use-context system:kube-controller-manager@kubernetes \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

生成kubernetes证书-scheduler

cd /root/k8s-ha-install/pki
#生成scheduler证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler

#设置集群项
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.169.1.200:8443 \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig

#设置环境项
kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig

#设置用户项
kubectl config set-credentials system:kube-scheduler \
--client-certificate=/etc/kubernetes/pki/scheduler.pem \
--client-key=/etc/kubernetes/pki/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig

#设置默认环境
kubectl config use-context system:kube-scheduler@kubernetes \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig

生成kubernetes证书-admin

cd /root/k8s-ha-install/pki
#生成scheduler证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin

#设置集群项
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.169.1.200:8443 \
--kubeconfig=/etc/kubernetes/admin.kubeconfig

#设置环境项
kubectl config set-context system:kube-admin@kubernetes \
--cluster=kubernetes \
--user=system:kube-admin \
--kubeconfig=/etc/kubernetes/admin.kubeconfig

#设置用户项
kubectl config set-credentials system:kube-admin \
--client-certificate=/etc/kubernetes/pki/admin.pem \
--client-key=/etc/kubernetes/pki/admin-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/admin.kubeconfig

#设置默认环境
kubectl config use-context system:kube-admin@kubernetes \
--kubeconfig=/etc/kubernetes/admin.kubeconfig

生成ServiceAccount Key

openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub

发送kubernetes证书到所有节点

MASTERNODES="k8s-master-202 k8s-master-203"
WORKNODES="k8s-worker-204 k8s-worker-205"
for NODE in $MASTERNODES; do
	echo $NODE
	ssh $NODE "mkdir -p /etc/kubernetes/pki/"
	for FLIE in $(ls /etc/kubernetes/pki/); do
  	scp /etc/kubernetes/pki/$FLIE $NODE:/etc/kubernetes/pki/$FLIE
	done
	for FLIE in "admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig"; do
  	scp /etc/kubernetes/$FLIE $NODE:/etc/kubernetes/$FLIE;
	done
done