k8s 1.23 + centos部署
kubernetes部署架构图
节点基础软件安装
环境准备
检查系统时区、时间
各个节点的系统时间要一致,否则导致证书等问题
参考 https://blog.csdn.net/xiao_yi_xiao/article/details/124308109
修改时区
# 修改时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# 保存
hwclock -w
修改时间
# 修改系统时间
date -s "2019-11-01 17:28:00"
# 修改硬件时间
hwclock --set --date "2019-11-01 17:28:00"
# 同步系统时间和硬件时间
hwclock --hctosys
# 保存
hwclock -w 或者 clock -w
# 重启系统
init 6
检查网络
使用命令ip route show,必须有default路由,没有手动添加
#route命令找不到,如果存在,忽略
yum -y install net-tools
#添加默认路由 gw后面接网关 dev后面是网卡名 公司内网没有网关,设本机ip即可
route add default gw 192.168.1.2 dev enp1s0
# 或修改
vim /etc/sysconfig/network-scripts/ifcfg-*
#添加如下两行,必须设置网关,可以直接用本机IP
#DEFROUTE=yes
#GATEWAY=x.x.x.x
systemctl restart network
机器名可能无法识别,需要在/etc/hosts添加
ip host-name
,比如127.0.0.1 k8s-master
安装docker或containerd
docker版本必须为20.10.7,containerd为1.4.6
这里需要安装与Kubernetes兼容的docker版本,参考链接:
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md
https://github.com/kubernetes/kubernetes/blob/v1.23.6/build/dependencies.yaml
containerd也需要和Docker兼容,参考链接:
https://docs.docker.com/engine/release-notes/
https://github.com/moby/moby/blob/v20.10.7/vendor.conf
安装docker
# 在 master 节点和 worker 节点都要执行
# 卸载旧版本
sudo yum remove -y docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-selinux \
docker-engine-selinux \
docker-engine \
docker-ce \
docker-ce-cli \
containerd.io
# 设置 yum repository
sudo yum install -y yum-utils \
device-mapper-persistent-data \
lvm2
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 安装并启动 docker,docker依赖containerd.io,不需要单独安装
sudo yum install -y docker-ce-20.10.7 docker-ce-cli-20.10.7 containerd.io-1.4.6
# 设置 docker 镜像,提高 docker 镜像下载速度和稳定性 推荐地址 http://f1361db2.m.daocloud.io/
# 如果您访问 https://hub.docker.io 速度非常稳定,亦可以跳过这个步骤
# 内网环境不需要,直接配置内网仓库即可
curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://f1361db2.m.daocloud.io
# 修改docker Cgroup Driver为systemd
#添加/etc/docker/daemon.json配置,添加
mkdir /etc/docker
cat <<EOF > /etc/docker/daemon.json
{
"registry-mirrors": ["http://f1361db2.m.daocloud.io"],
"insecure-registries": ["192.168.1.2:5000"],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl daemon-reload
sudo systemctl enable docker
sudo systemctl start docker
# 检查 docker 版本
docker info
安装nfs-utils
# 在 master 节点和 worker 节点都要执行
sudo yum install -y nfs-utils
安装kubectl/kubeadm/kubelet
部署kubectl / kubeadm / kubelet(所有节点均需安装)
# 关闭 防火墙
systemctl stop firewalld
systemctl disable firewalld
# 关闭 SeLinux
setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
# 关闭 swap
swapoff -a
yes | cp /etc/fstab /etc/fstab_bak
cat /etc/fstab_bak |grep -v swap > /etc/fstab
# bridged网桥设置 (containerd要执行的,使用docker不需要)
# 新建modules-load.d/k8s.conf文件
cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
# 新建sysctl.d/k8s.conf文件
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
#加载配置文件
sysctl --system
#加载br_netfilter网桥过滤模块,和加载网络虚拟化技术模块
modprobe br_netfilter
modprobe overlay
#检验网桥过滤模块是否加载成功
lsmod | grep -e br_netfilter -e overlay
# 配置K8S的yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 安装kubelet、kubeadm、kubectl
yum install -y kubelet-1.23.9 kubeadm-1.23.9 kubectl-1.23.9
# 重启 docker,并启动 kubelet
systemctl restart docker
systemctl enable kubelet && systemctl start kubelet
master节点部署
下载镜像
查看集群所需镜像的版本
kubeadm config images list
编辑镜像下载文件 images.sh,然后执行。其中node节点只需要kube-proxy和pause
tee ./images.sh <<'EOF'
#!/bin/bash
images=(
kube-apiserver:v1.23.9
kube-controller-manager:v1.23.9
kube-scheduler:v1.23.9
kube-proxy:v1.23.9
pause:3.6
etcd:3.5.1-0
coredns:v1.8.6
)
for imageName in ${images[@]} ; do
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName
done
EOF
# 拉取镜像
chmod +x ./images.sh && ./images.sh
初始化master节点
查看默认配置
kubeadm config print init-defaults
# 1.23.17
registry.k8s.io/kube-apiserver:v1.23.17
registry.k8s.io/kube-controller-manager:v1.23.17
registry.k8s.io/kube-scheduler:v1.23.17
registry.k8s.io/kube-proxy:v1.23.17
registry.k8s.io/pause:3.6
registry.k8s.io/etcd:3.5.6-0
registry.k8s.io/coredns/coredns:v1.8.6 registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.6
#导出镜像到本地命令
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.23.17 > kube-apiserver_v1.23.17.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.23.17 > kube-controller-manager_v1.23.17.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.23.17 > kube-scheduler_v1.23.17.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.23.17 > kube-proxy_v1.23.17.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6 > pause_3.6.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.6-0 > etcd_3.5.6-0.image
docker save registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.6 > coredns_v1.8.6.image
#calico
docker save calico/cni:v3.25.0 > calico_cni_v3.25.0.image
docker save calico/node:v3.25.0 > calico_node_v3.25.0.image
docker save calico/kube-controllers:v3.25.0 > calico_kube-controllers_v3.25.0.image
执行初始化
# master节点上执行 --apiserver-advertise-address=192.168.1.2 \
kubeadm init \
--apiserver-advertise-address=192.168.1.2 \
--control-plane-endpoint=k8s-master-001 \
--image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version=v1.23.9 \
--service-cidr=10.96.0.0/16 \
--pod-network-cidr=10.100.0.0/16 \
--upload-certs \
--v=10
配置 kubectl
rm -rf /root/.kube/
mkdir /root/.kube/
cp -i /etc/kubernetes/admin.conf /root/.kube/config
检查状态
#检查 Master 状态
kubectl cluster-info
#查看节点状态
kubectl get nodes -o wide
**注意:**如果是INTERNAL-IP都是相同的或为空(VirtualBox下安装的)都是10.0.2.15 的IP则需要手动修改配置文件/etc/sysconfig/kubelet
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kubelet/
KUBELET_EXTRA_ARGS=--node-ip=192.168.1.2 # 改成节点真实ip
出现错误,重置环境
swapoff -a && kubeadm reset -f && systemctl daemon-reload && systemctl restart kubelet && iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X && systemctl restart docker
网络插件安装
calico
非operator方式,参考 https://blog.csdn.net/yy8623977/article/details/124707433
# 安装 calico 网络插件
# 参考文档 https://projectcalico.docs.tigera.io/getting-started/kubernetes/self-managed-onprem/onpremises
echo "安装calico-3.25"
wget https://docs.projectcalico.org/archive/v3.25/manifests/calico.yaml
## 放开下面两行,并修改为kubeadm init中设置的pod网格 10.100.0.0/16
# - name: CALICO_IPV4POOL_CIDR
# value: "192.168.0.0/16"
#查看获取镜像
cat calico.yaml | grep image
#手动获取镜像
tee ./calicoImages.sh <<'EOF'
#!/bin/bash
images=(
docker.io/calico/cni:v3.25.0
docker.io/calico/node:v3.25.0
docker.io/calico/kube-controllers:v3.25.0
)
for imageName in ${images[@]} ; do
docker pull $imageName
done
EOF
#获取镜像
chmod +x ./calicoImages.sh && ./calicoImages.sh
#部署calico
kubectl apply -f calico.yaml
kubernetes dashboard安装
https://blog.csdn.net/yy8623977/article/details/124707433
1.创建Dashboard的yaml文件
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.1/aio/deploy/recommended.yaml
2.使用如下命令或直接手动编辑kubernetes-dashboard.yaml文件
sed -i '/targetPort:/a\ \ \ \ \ \ nodePort: 30001\n\ \ type: NodePort' kubernetes-dashboard.yaml
#注释 imagePullPolicy: Always 避免每次都需要重复拉取镜像
3.创建pod,service
需kubernetesui/dashboard:v2.5.1、kubernetesui/metrics-scraper:v1.0.7两个镜像(已提供离线镜像,手动导入)
kubectl create -f kubernetes-dashboard.yaml
4.创建完后检测
kubectl get deployment kubernetes-dashboard -n kubernetes-dashboard
kubectl get pods -n kubernetes-dashboard -o wide
kubectl get services -n kubernetes-dashboard
netstat -ntlp|grep 30001
5.创建账户
创建dashboard-account.yaml
tee ./dashboard-account.yaml <<'EOF'
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
EOF
部署dashboard-account.yaml
kubectl apply -f dashboard-account.yaml
6.获取token
kubectl -n kubernetes-dashboard get secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}"
解决浏览器证书问题
https://blog.csdn.net/sshwolf/article/details/119578839
openssl genrsa -out dashboard.key 2048
openssl req -days 3650 -new -out dashboard.csr -key dashboard.key -subj '/CN=192.168.1.2'
openssl x509 -req -in dashboard.csr -signkey dashboard.key -out dashboard.crt
kubectl delete secret kubernetes-dashboard-certs -n kubernetes-dashboard
kubectl create secret generic kubernetes-dashboard-certs --from-file=dashboard.key --from-file=dashboard.crt -n kubernetes-dashboard
metrics-server部署
参考https://blog.csdn.net/yy8623977/article/details/124872606
下载https://github.com/kubernetes-sigs/metrics-server/releases,当前版本v0.6.1,components.yaml为普通版本,high-availability.yaml为高可用版本
修改配置
args:
- --cert-dir=/tmp
- --secure-port=4443
# - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
# 注释上面那行,添加下面两行
- --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP # node address类型的优先权设置
- --kubelet-insecure-tls # 取消安全验证
- --kubelet-use-node-status-port
- --metric-resolution=15s
# image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1
# 注释上面那行,添加下面这行
image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
应用部署文件
kubectl apply -f components.yaml
work节点加入集群
(保证与master时间一致)
拉镜像,或手动导入镜像
# worker节点需要的镜像
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.23.9
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6
docker pull docker.io/calico/node:v3.19.4
docker pull docker.io/calico/cni:v3.19.4
docker pull docker.io/calico/pod2daemon-flexvol:v3.19.4
加入集群
# 在 master 节点执行 获取kubeadm join命令
kubeadm token create --print-join-command
# worker节点上执行
kubeadm join k8s-master-001:6443 --token j3h4iv.ojdq3fbfgzcklw3m \
--discovery-token-ca-cert-hash sha256:7ad81dad772080e20f4793e6cae67cc6c00c007867d94f6be2cbad3d703a0cab \
--control-plane --certificate-key b2f4a2db18e580830daf9a3d42fd0683f91e6f4ede653d105139b9f392cf5dac