kubernetes多节点集群搭建过程分析

前端时间在研究k8s+jenkins+harbor环境搭建，在此把搭建过程分享一下。

我只想介绍K8s集群的搭建。

配置说明  四台虚拟机 IP地址为 71，72，73，74 

71、74为Master节点  72，73为node节点  好了下边开始搭建了

初始化环境

1、修改hostname 四个服务器分别为k8smaster1 k8smaster2 k8snode1 k8snode2
     

hostnamectl set-hostname 主机名
vim /etc/hostname 修改名称
 hostname 主机名

2、关闭防火墙
 

[root@XXX ~]# systemctl stop firewalld 
[root@XXX ~]# systemctl disable firewalld 
# 确认是否运行
[root@XXX ~]# firewall-cmd --state not running

 

3、关闭Seliunx

 

[root@XXX ~]# sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
[root@master local]# cat /etc/selinux/config 

# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected. 
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted

 

4、设置集群时间同步

#修改时区
rm -rf /etc/localtime
ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
vim /etc/sysconfig/clock
ZONE="Asia/Shanghai"
UTC=false
ARC=false

#安装并设置开机自启
yum install -y ntp
systemctl start ntpd
systemctl enable ntpd

#配置开机启动校验
vim /etc/rc.d/rc.local
/usr/sbin/ntpdate ntp1.aliyun.com > /dev/null 2>&1; /sbin/hwclock -w

#配置定时任务
crontab -e
0 */1 * * * ntpdate ntp1.aliyun.com > /dev/null 2>&1; /sbin/hwclock -w

5、关闭swap分区

打开编辑并注释掉相关内容
[root@node2 local]# vim /etc/fstab
# /etc/fstab
# Created by anaconda on Wed Sep 16 18:50:24 2020
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/centos-root /                       xfs     defaults        0 0
UUID=71a3a2c7-1e60-4bc6-b641-8e82b3d1e79b /boot                   xfs     defaults        0 0
#/dev/mapper/centos-swap swap                    swap    defaults        0 0

#保存，退出

#使用命令查看，此时是还有的，因为没有重启
[root@node2 local]# free -m
              total        used        free      shared  buff/cache   available
Mem:           3770         138        3456          11         175        3421
Swap:          2047           0        2047

# 重启
[root@node2 local]# reboot

# 重启完毕后再次查看
[root@node1 ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:           3770         134        3448          11         187        3419
Swap:             0           0           0

 

6、添加网桥过滤

# 添加网桥过滤及地址转发
[root@master ~]# vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1 
net.bridge.bridge-nf-call-iptables = 1 
net.ipv4.ip_forward = 1 
vm.swappiness = 0

# 加载br_netfilter模块
[root@master ~]# modprobe br_netfilter
[root@master ~]# lsmod | grep br_netfilter
br_netfilter           22256  0 
bridge                151336  1 br_netfilter

# 加载网桥过滤配置文件
[root@master ~]# sysctl -p /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0

7、开启IPvs

# 安装ipset以及ipvsadm 
[root@master ~]# yum -y install ipset ipvsadm

#添加需要加载的模块（直接复制下面所有内容，粘贴到命令行中）
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash 
modprobe -- ip_vs 
modprobe -- ip_vs_rr 
modprobe -- ip_vs_wrr 
modprobe -- ip_vs_sh 
modprobe -- nf_conntrack_ipv4 
EOF

#验证一下
[root@master ~]# ll /etc/sysconfig/modules/
总用量 4
-rw-r--r-- 1 root root 130 11月  4 15:22 ipvs.modules

# 修改权限
[root@master ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules 

#执行
[root@master ~]# sh /etc/sysconfig/modules/ipvs.modules

#验证其中一个
[root@master ~]# lsmod | grep ip_vs_rr
ip_vs_rr               12600  0 
ip_vs                 145497  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr

 

 

---

安装Docker-Ce

参照官网给出的安装步骤

#clear yum 
$ sudo yum remove docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-engine
#安装工具包
$ sudo yum install -y yum-utils
$ sudo yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo
#安装docker
$ sudo yum install docker-ce docker-ce-cli containerd.io
$ yum list docker-ce --showduplicates | sort -r

docker-ce.x86_64  3:18.09.1-3.el7                     docker-ce-stable
docker-ce.x86_64  3:18.09.0-3.el7                     docker-ce-stable
docker-ce.x86_64  18.06.1.ce-3.el7                    docker-ce-stable
docker-ce.x86_64  18.06.0.ce-3.el7                    docker-ce-stable
$ sudo yum install docker-ce-<VERSION_STRING> docker-ce-cli-<VERSION_STRING> containerd.io
$ sudo systemctl start docker
$ sudo systemctl enale docker
#测试
$ sudo docker run hello-world

 

---

部署软件及配置：

所有k8s集群节点均需安装，默认yum源是谷歌，可以使用阿里云yum

需求	kubeadm	kubelet	kubectl	docker-ce
值	初始化集群、管理集群等，版本为：1.17.2	用于接收api-server指令，对pod生命周期进行管理，版本为：1.17.2	集群命令行管理工具，版本为：1.17.2	18.06.3

# 谷歌yum源
[kubernetes] 
name=Kubernetes 
baseurl=https://packages.cloud.google.com/yum /repos/kubernetes-el7-x86_64 
enabled=1 
gpgcheck=1 
repo_gpgcheck=1 
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

# 阿里云yum源
[kubernetes]
name=Kubernetes 
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/  
enabled=1 
gpgcheck=1 
repo_gpgcheck=1 
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg  
       https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

# 新建文件（每台机器）
[root@master ~]# vim /etc/yum.repos.d/k8s.repo
[kubernetes]
name=Kubernetes 
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/  
enabled=1 
gpgcheck=1 
repo_gpgcheck=1 
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg  
       https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

:wq 保存退出


# 检查yum源是否可用（每台机器）
[root@master ~]# yum list | grep kubeadm
导入 GPG key 0xA7317B0F:
 用户ID     : "Google Cloud Packages Automatic Signing Key <gc-team@google.com>"
 指纹       : d0bc 747f d8ca f711 7500 d6fa 3746 c208 a731 7b0f
 来自       : https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
y  
kubeadm.x86_64                              1.19.3-0                   kubernetes

# 安装
[root@master ~]# yum -y install kubeadm kubelet kubectl

主要配置kubelet，如果不配置可能会导致k8s集群无法启动

# 为了实现docker使用的cgroupdriver与kubelet使用的 cgroup的一致性，建议修改如下文件内容。 
[root@XXX ~]# vim /etc/sysconfig/kubelet 
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

# 设置为开机启动，注意:这里千万不要去手动启动它，它的启动是由kubeadm初始化的时候启动
[root@master ~]# systemctl enable kubelet
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kube
let.service.

初始化操作

kubeadm init   --kubernetes-version=v1.19.4     --apiserver-advertise-address=10.10.11.71  --control-plane-endpoint 10.10.11.71:6443 --image-repository registry.aliyuncs.com/google_containers     --pod-network-cidr=192.168.0.0/16 



Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 10.10.11.71:6443 --token c5arar.09aoifdbmbwykrc8 \
    --discovery-token-ca-cert-hash sha256:80fbd0fd738febd721d98f4881a19fbe6c5f74e044bed65ee4cac35fd1c1c815 \
    --control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.10.11.71:6443 --token c5arar.09aoifdbmbwykrc8 \
    --discovery-token-ca-cert-hash sha256:80fbd0fd738febd721d98f4881a19fbe6c5f74e044bed65ee4cac35fd1c1c815 

按照要求执行命令

mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

配置Calico资源配置清单文件

cd /usr/local/k8s/reslist
kubectl apply -f calico.yaml

kubectl get nodes 

 

添加第二个Master节点

scp root@10.10.11.71:/etc/kubernetes/pki/ca.* /etc/kubernetes/pki/
scp root@10.10.11.71:/etc/kubernetes/pki/sa.* /etc/kubernetes/pki/
scp root@10.10.11.71:/etc/kubernetes/pki/front-proxy-ca.* /etc/kubernetes/pki/
scp root@10.10.11.71:/etc/kubernetes/pki/etcd/ca.* /etc/kubernetes/pki/etcd/
scp root@10.10.11.71/etc/kubernetes/admin.conf /etc/kubernetes/

 kubeadm join 10.10.11.71:6443 --token c5arar.09aoifdbmbwykrc8 \
    --discovery-token-ca-cert-hash sha256:80fbd0fd738febd721d98f4881a19fbe6c5f74e044bed65ee4cac35fd1c1c815 \
    --control-plane 

添加Node节点

kubeadm join 10.10.11.71:6443 --token c5arar.09aoifdbmbwykrc8 \
    --discovery-token-ca-cert-hash sha256:80fbd0fd738febd721d98f4881a19fbe6c5f74e044bed65ee4cac35fd1c1c815