k8s集群

环境初始化

# 重命名
[root@localhost ~]# hostnamectl set-hostname master1
[root@localhost ~]# su
su
#配置静态IP
[root@master1 ~]# cd /etc/sysconfig/network-scripts/
[root@master1 network-scripts]# vim ifcfg-ens33 

BOOTPROTO="none"
NAME="ens33"
DEVICE="ens33"
ONBOOT="yes"
IPADDR=192.168.136.161
PREFIX=24
GATEWAY=192.168.136.2
DNS1=114.114.114.114
DNS2=101.226.4.6


[root@master1 network-scripts]# service network restart
Restarting network (via systemctl):                        [  确定  ]
#检查IP地址
[root@master1 network-scripts]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:2f:4d:2c brd ff:ff:ff:ff:ff:ff
    inet 192.168.136.161/24 brd 192.168.136.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe2f:4d2c/64 scope link 
       valid_lft forever preferred_lft forever
#查看路由
[root@master1 network-scripts]# ip route
default via 192.168.136.2 dev ens33 proto static metric 100 
192.168.136.0/24 dev ens33 proto kernel scope link src 192.168.136.153 metric 100 
#查看本地DNS服务器
[root@master1 network-scripts]# cat /etc/resolv.conf 
# Generated by NetworkManager
nameserver 114.114.114.114
nameserver 101.226.4.6

配置hosts文件

vim /etc/hosts
cat /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.136.161  master1
192.168.136.162  master2
192.168.136.164  node1
192.168.136.165  node2
192.168.136.166  node3

更新和配置软件源

[root@master1 network-scripts]# cd /etc/yum.repos.d/
[root@master1 yum.repos.d]# rm -f *
[root@master1 yum.repos.d]# curl -O http://mirrors.aliyun.com/repo/Centos-7.repo
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2523  100  2523    0     0  30244      0 --:--:-- --:--:-- --:--:-- 30768
[root@master1 yum.repos.d]# ls
Centos-7.repo
[root@master1 yum.repos.d]# yum makecache fast
已加载插件：fastestmirror
Determining fastest mirrors
 * base: mirrors.aliyun.com
 * extras: mirrors.aliyun.com
 * updates: mirrors.aliyun.com

关闭firewalld和selinux

[root@master1 yum.repos.d]# systemctl stop firewalld
[root@master1 yum.repos.d]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
#临时关闭selinux
[root@master1 yum.repos.d]# setenforce 0
#永久关闭
[root@master1 yum.repos.d]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
[root@master1 yum.repos.d]# reboot
[root@master1 ~]# getenforce
Disabled

关闭交换分区

#临时关闭
[root@master1 ~]# swapoff -a
#永久关闭
[root@master1 ~]# vim /etc/fstab 
#/dev/mapper/centos-swap swap                    swap    defaults        0 0

调整内核参数

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

sysctl -p
modprobe br_netfilter
lsmod | grep br_netfilter

配置ipvs功能

# 安装ipset和ipvsadm
yum install ipset ipvsadmin -y
# 添加需要加载的模块写入脚本文件
cat <<EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# 为脚本文件添加执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules
# 执行脚本文件
/bin/bash /etc/sysconfig/modules/ipvs.modules
# 查看对应的模块是否加载成功
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
# 重启
reboot

配置时间同步

# 跟网络时间做同步
ntpdate ntp.cloud.aliyuncs.com
# 添加计划任务
crontab -e 
* */1 * * * /usr/sbin/ntpdate ntp.cloud.aliyuncs.com
# 重启crond服务
service crond restart

配置docker环境

下载安装docker的仓库文件

cd /etc/yum.repos.d/
curl -O https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

ls
Centos-7.repo  docker-ce.repo

安装docker环境

yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

#查看docker版本
[root@master1 yum.repos.d]# docker -v
Docker version 26.1.4, build 5650f9b
#启动docker
[root@master1 yum.repos.d]# systemctl start docker
#设置开机自启
[root@master1 yum.repos.d]# systemctl enable docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
#查看docker状态
[root@master1 yum.repos.d]# systemctl status docker

配置docker镜像加速

[root@master1 yum.repos.d]# vim /etc/docker/daemon.json

{
  "registry-mirrors": ["https://hub.docker-alhk.dkdun.com/"],
  "exec-opts": ["native.cgroupdriver=systemd"]       #指定cgroup的驱动程序是systemd
}

#重新加载docker的配置文件和重启docker服务
[root@master1 yum.repos.d]# systemctl daemon-reload
[root@master1 yum.repos.d]# systemctl restart docker

配置cri-docker

kubernets 1.24版本后默认使用containerd做底层容器，需要使用cri-dockerd做中间层来与docker通信

mkdir /cri-docker
cd /cri-docker/
# 下载
wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.8/cri-dockerd-0.3.8-3.el7.x86_64.rpm
# 安装
rpm -ivh cri-dockerd-0.3.8-3.el7.x86_64.rpm
# 重载系统守护进程
systemctl daemon-reload
# 修改配置文件
vim /usr/lib/systemd/system/cri-docker.service
# 修改第10行 ExecStart
# 改为
ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint fd://

配置cri-docker服务自启动

# 重载系统守护进程
systemctl daemon-reload
# 启动cri-dockerd
systemctl start cri-docker.socket cri-docker
# 设置cri-dockerd自启动
systemctl enable cri-docker.socket cri-docker
# 检查Docker组件状态
systemctl status docker cir-docker.socket cri-docker

配置k8s集群环境

安装kubectl

# 下载
curl -LO "https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl"
# 检验 可选
curl -LO "https://dl.k8s.io/v1.28.2/bin/linux/amd64/kubectl.sha256"
echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
# 安装
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
# 测试
kubectl version --client

#Client Version: v1.28.2
#Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3

配置k8s组件源

cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum makecache

安装

# 安装
yum install -y install kubeadm-1.28.2-0 kubelet-1.28.2-0 kubectl-1.28.2-0 --disableexcludes=kubernetes
# 如果报错未找到就试试不指定版本
yum install -y install kubeadm kubelet kubectl --disableexcludes=kubernetes
# 设置自启动
systemctl enable --now kubelet

集群初始化

在master节点执行

kubeadm init --kubernetes-version=v1.28.2 \
	--pod-network-cidr=10.224.0.0/16 \
	--apiserver-advertise-address=192.168.136.161 \
	--image-repository=registry.aliyuncs.com/google_containers \
	--cri-socket=unix:///var/run/cri-dockerd.sock
	
#192.168.136.161为master1的IP

swap 未关闭报错

kubectl 未启动报错
之后启动后再执行命令还是报错，端口 6443, 10259, 10257, 10250, 2379, 2380 都被占用了，有写yaml文件已存在， etcd 数据目录不为空
ps aux |grep 6443
kill -9 3058 
......
删除已存在的yaml文件
rm /etc/kubernetes/manifests/kube-apiserver.yaml  
rm /etc/kubernetes/manifests/kube-controller-manager.yaml  
rm /etc/kubernetes/manifests/kube-scheduler.yaml  
rm /etc/kubernetes/manifests/etcd.yaml
清空 etcd 数据目录
rm -rf /var/lib/etcd/*

成功后会提示以下信息：

[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.136.161:6443 --token rokdqu.7bphgp43lhf0tteu \
	--discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e 

记下系统提示命令kubeadm join xxxxx，并在后面追加unix:///var/run/cri-dockerd.sock

完整命令应该类似于：kubeadm join 192.168.136.161:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --cri-socket unix:///var/run/cri-dockerd.sock

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

查找token和certificate key

[root@master1 .kube]# kubeadm init phase upload-certs --upload-certs
I0816 16:16:42.129855    8453 version.go:256] remote version is much newer: v1.31.0; falling back to: stable-1.28
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
b5ecc6f24110ac0e31a1026a74fe95ad4e2fdab1c6dba919b7f651c9d7dd265f
[root@master1 .kube]# kubeadm token create --print-join-command
kubeadm join 192.168.136.161:6443 --token btyzsw.3o1zswwdm0v904pr --discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e 

node节点加入集群

# 上面得到的命令
kubeadm join 192.168.136.161:6443 --token rokdqu.7bphgp43lhf0tteu \
	--discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e \
        --cri-socket unix:///var/run/cri-dockerd.sock
        
        
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

master节点加入集群

#参考文档
#https://kubernetes.io/zh-cn/docs/reference/setup-tools/kubeadm/kubeadm-join/
#在node节点加入集群命令中添加参数  --control-plane

kubeadm join 192.168.136.161:6443 --token rokdqu.7bphgp43lhf0tteu \
	--discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e \
	--control-plane  \
        --cri-socket unix:///var/run/cri-dockerd.sock

出现下面的错误，集群当前没有配置稳定的 controlPlaneEndpoint 地址

这通常是在集群初始化时通过 kubeadm init 命令的 --control-plane-endpoint 参数来设置的。

[root@master2 ~]# kubeadm join 192.168.136.161:6443 --token rokdqu.7bphgp43lhf0tteu \
> --discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e \
> --control-plane  \
>         --cri-socket unix:///var/run/cri-dockerd.sock
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
error execution phase preflight: 
One or more conditions for hosting a new control plane instance is not satisfied.

unable to add a new control plane instance to a cluster that doesn't have a stable controlPlaneEndpoint address

Please ensure that:
* The cluster has a stable controlPlaneEndpoint address.
* The certificates that must be shared among control plane instances are provided.


To see the stack trace of this error execute with --v=5 or higher

解决方法

[root@master1 ~]# kubectl get cm kubeadm-config -n kube-system
NAME             DATA   AGE
kubeadm-config   1      33m
[root@master1 ~]# kubectl describe cm kubeadm-config -n kube-system
Name:         kubeadm-config
Namespace:    kube-system
Labels:       <none>
Annotations:  <none>

Data
====
ClusterConfiguration:
----
apiServer:
  extraArgs:
    authorization-mode: Node,RBAC
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.28.2
networking:
  dnsDomain: cluster.local
  podSubnet: 10.224.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}


BinaryData
====

Events:  <none>
[root@master1 ~]# kubectl edit cm kubeadm-config -n kube-system
configmap/kubeadm-config edited

master2执行命令加入节点

kubeadm join 192.168.136.161:6443 --token rokdqu.7bphgp43lhf0tteu --discovery-token-ca-cert-hash sha256:c0b81b87cd45cd030bf7674b3d25a2d06dbd20ea78817f2461e9ddcbcf1c7f2e  --control-plane --certificate-key  b5ecc6f24110ac0e31a1026a74fe95ad4e2fdab1c6dba919b7f651c9d7dd265f  --cri-socket unix:///var/run/cri-dockerd.sockkube

完成后查看节点

[root@master1 ~]# kubectl get node
NAME      STATUS     ROLES           AGE     VERSION
master1   NotReady   control-plane   119m    v1.28.2
master2   NotReady   control-plane   7m49s   v1.28.2
node1     NotReady   <none>          107m    v1.28.2
node2     NotReady   <none>          107m    v1.28.2
node3     NotReady   <none>          107m    v1.28.2

分配worker

# 在master上执行
kubectl label node node1 node-role.kubernetes.io/worker=worker
kubectl label node node2 node-role.kubernetes.io/worker=worker
kubectl label node node3 node-role.kubernetes.io/worker=worker

安装Calico网络插件

# master执行
wget https://docs.projectcalico.org/manifests/calico.yaml
kubectl apply -f calico.yaml
# 验证
kubectl get nodes

NAME      STATUS   ROLES           AGE    VERSION
master1   Ready    control-plane   126m   v1.28.2
master2   Ready    control-plane   14m    v1.28.2
node1     Ready    worker          114m   v1.28.2
node2     Ready    worker          114m   v1.28.2
node3     Ready    worker          114m   v1.28.2

#查看pod
root@master1 ~]# kubectl get pod -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS       AGE
kube-system   calico-kube-controllers-658d97c59c-prwkv   1/1     Running   0              5m25s
kube-system   calico-node-2kdfk                          1/1     Running   0              5m25s
kube-system   calico-node-47hcn                          1/1     Running   0              5m25s
kube-system   calico-node-4pc5c                          1/1     Running   0              5m25s
kube-system   calico-node-nsqfv                          1/1     Running   0              5m25s
kube-system   calico-node-vltbx                          1/1     Running   0              5m25s
kube-system   coredns-66f779496c-k2hf8                   1/1     Running   0              127m
kube-system   coredns-66f779496c-sr9rc                   1/1     Running   0              127m
kube-system   etcd-master1                               1/1     Running   1 (129m ago)   127m
kube-system   etcd-master2                               1/1     Running   0              15m
kube-system   kube-apiserver-master1                     1/1     Running   1 (130m ago)   127m
kube-system   kube-apiserver-master2                     1/1     Running   0              15m
kube-system   kube-controller-manager-master1            1/1     Running   2 (15m ago)    127m
kube-system   kube-controller-manager-master2            1/1     Running   0              15m
kube-system   kube-proxy-7w9qw                           1/1     Running   0              15m
kube-system   kube-proxy-8bb5g                           1/1     Running   0              115m
kube-system   kube-proxy-b8r8z                           1/1     Running   0              115m
kube-system   kube-proxy-cbhx4                           1/1     Running   0              127m
kube-system   kube-proxy-dg65j                           1/1     Running   0              115m
kube-system   kube-scheduler-master1                     1/1     Running   2 (15m ago)    127m
kube-system   kube-scheduler-master2                     1/1     Running   0              15m

安装Dashboard

以下命令均只在master节点上执行

下载安装

wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml

vim recommended.yaml

修改Service部分，改为NodePort对外暴露端口

The range of valid ports is 30000-32767

39 spec:
40   type: NodePort
41   ports:
42     - port: 443
43       targetPort: 8443
44       nodePort: 30081
45   selector:
46     k8s-app: kubernetes-dashboard

安装

kubectl apply -f recommended.yaml

查看

[root@master1 ~]# kubectl get pods,svc -n kubernetes-dashboard
NAME                                             READY   STATUS    RESTARTS   AGE
pod/dashboard-metrics-scraper-5657497c4c-zlfz6   1/1     Running   0          88s
pod/kubernetes-dashboard-78f87ddfc-9zrss         1/1     Running   0          88s

NAME                                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
service/dashboard-metrics-scraper   ClusterIP   10.99.245.129   <none>        8000/TCP        88s
service/kubernetes-dashboard        NodePort    10.98.7.173     <none>        443:30081/TCP   12s

创建账号

创建dashboard-access-token.yaml文件

# Creating a Service Account
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
# Creating a ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
---
# Getting a long-lived Bearer Token for ServiceAccount
apiVersion: v1
kind: Secret
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
  annotations:
    kubernetes.io/service-account.name: "admin-user"
type: kubernetes.io/service-account-token
# Clean up and next steps
# kubectl -n kubernetes-dashboard delete serviceaccount admin-user
# kubectl -n kubernetes-dashboard delete clusterrolebinding admin-user

执行

kubectl apply -f dashboard-access-token.yaml
#获取token
kubectl get secret admin-user -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d

访问dashboard

[root@master1 ~]# kubectl get secret -n kubernetes-dashboard
NAME                              TYPE                                  DATA   AGE
admin-user                        kubernetes.io/service-account-token   3      49s
kubernetes-dashboard-certs        Opaque                                0      4m58s
kubernetes-dashboard-csrf         Opaque                                1      4m58s
kubernetes-dashboard-key-holder   Opaque                                2      4m58s
[root@master1 ~]# kubectl get svc -n kubernetes-dashboard
NAME                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
dashboard-metrics-scraper   ClusterIP   10.99.245.129   <none>        8000/TCP        5m20s
kubernetes-dashboard        NodePort    10.98.7.173     <none>        443:30081/TCP   4m4s

浏览器访问集群ip:端口(https://192.168.136.161:30081/)，注意https

输入上一步获取到的token即可

解决token默认15分钟过期的问题

[root@k8s-master-1 ~]# vim recommended.yaml 

193       containers:
194         - name: kubernetes-dashboard
195           image: kubernetesui/dashboard:v2.7.0
196           imagePullPolicy: Always
197           ports:
198             - containerPort: 8443
199               protocol: TCP
200           args:
201             - --auto-generate-certificates
202             - --namespace=kubernetes-dashboard
203             - --token-ttl=43200         #添加这条配置，超时时间调整为12小时
重新应用
[root@k8s-master-1 ~]# kubectl apply  -f recommended.yaml 

安装kubectl命令自动补全

yum install bash-completion -y
# 临时设置自动补全
source <(kubectl completion bash) 
# 永久设置自动补全
echo "source <(kubectl completion bash)" >> ~/.bashrc && bash

部署metric-server

下载

master执行

wget https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.2/components.yaml

修改

vim修改140行左右

原：

containers:
- args:
  ...
  image: k8s.gcr.io/metrics-server/metrics-server:v0.6.2

修改后：

containers:
- args:
  ...
  - --kubelet-insecure-tls  # 添加这一行
  image: admin4j/metrics-server:v0.6.2  # 修改镜像仓库地址

应用

kubectl apply -f components.yaml

查看

[root@master1 ~]# kubectl top nodes
NAME      CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master1   180m         9%     1143Mi          66%       
master2   147m         7%     1050Mi          61%       
node1     78m          3%     866Mi           50%       
node2     79m          3%     894Mi           52%       
node3     81m          4%     914Mi           53%   

目前问题，master1和master2都需要启动，否则会报错

#master1上报错
[root@master1 ~]# kubectl get   nodes
Error from server: etcdserver: request timed out

#master2上报错
root@master2 ~]# kubectl get pod -o wide -A
Unable to connect to the server: dial tcp 192.168.136.161:6443: i/o timeout

Prometheus监控k8s

监控方案

Cadvisor + node-exporter + prometheus + grafana

• Cadvisor：数据采集
• node-exporter：汇总
• prometheus：处理、存储
• grafana：展示

监控流程

• 容器监控：Prometheus使用cadvisor采集容器监控指标，而cadvisor集成在K8S的kubelet中所以无需部署，通过Prometheus进程存储，使用grafana进行展示。
• node节点监控：node端的监控通过node_exporter采集当前主机的资源，通过Prometheus进程存储，最后使用grafana进行展示。
• master节点监控：master的监控通过kube-state-metrics插件从K8S获取到apiserver的相关数据并通过网页页面暴露出来，然后通过Prometheus进程存储，最后使用grafana进行展示

#上传yaml文件
[root@master1 prometheus-k8s]# ls
configmap.yaml  csdn—prometheus监控k8s.txt  grafana-deploy.yaml  grafana-ing.yaml  grafana-svc.yaml  node-exporter.yaml  prometheus.deploy.yml  prometheus.svc.yml  rbac-setup.yaml


#修改prometheus.deploy.yml,镜像版本修改为 - image: prom/prometheus
#node-exporter.yaml，prometheus.deploy.yml，grafana-deploy.yaml
#提前在节点拉取镜像

docker pull prom/node-exporter
docker pull prom/prometheus
docker pull grafana/grafana:6.1.4

采用daemonset方式部署node-exporter

[root@master1 prometheus-k8s]# kubectl apply -f node-exporter.yaml 
daemonset.apps/node-exporter created
service/node-exporter created

[root@master1 prometheus-k8s]# kubectl get pods -A -o wide |grep exporter
kube-system            node-exporter-5kfl9                          1/1     Running   0             60s   10.224.135.7      node3     <none>           <none>
kube-system            node-exporter-ckg9f                          1/1     Running   0             60s   10.224.104.3      node2     <none>           <none>
kube-system            node-exporter-s2frk                          1/1     Running   0             60s   10.224.166.131    node1     <none>           <none>

[root@master1 prometheus-k8s]# kubectl get daemonset -A
NAMESPACE     NAME            DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE
kube-system   calico-node     5         5         5       5            5           kubernetes.io/os=linux   18h
kube-system   kube-proxy      5         5         5       5            5           kubernetes.io/os=linux   20h
kube-system   node-exporter   3         3         3       3            3           <none>                   2m41s
[root@master1 prometheus-k8s]# kubectl get svc -A
NAMESPACE              NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
default                kubernetes                  ClusterIP   10.96.0.1        <none>        443/TCP                  20h
kube-system            kube-dns                    ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP   20h
kube-system            metrics-server              ClusterIP   10.103.190.182   <none>        443/TCP                  17h
kube-system            node-exporter               NodePort    10.99.110.5      <none>        9100:31672/TCP           2m48s
kubernetes-dashboard   dashboard-metrics-scraper   ClusterIP   10.99.245.129    <none>        8000/TCP                 18h
kubernetes-dashboard   kubernetes-dashboard        NodePort    10.98.7.173      <none>        443:30081/TCP            18h

部署Prometheus

[root@master1 prometheus-k8s]# kubectl apply -f rbac-setup.yaml
clusterrole.rbac.authorization.k8s.io/prometheus created
serviceaccount/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
[root@master1 prometheus-k8s]# kubectl apply -f configmap.yaml
configmap/prometheus-config created
[root@master1 prometheus-k8s]# kubectl apply -f prometheus.deploy.yml
deployment.apps/prometheus created
[root@master1 prometheus-k8s]# kubectl apply -f prometheus.svc.yml
service/prometheus created

部署grafana

[root@master1 prometheus-k8s]# kubectl apply -f grafana-deploy.yaml
deployment.apps/grafana-core created
[root@master1 prometheus-k8s]# kubectl apply -f grafana-svc.yaml
service/grafana created
[root@master1 prometheus-k8s]# kubectl apply -f grafana-ing.yaml
[root@master1 prometheus-k8s]# cd ..
[root@master1 k8s-prometheus-grafana-master]# kubectl apply -f grafana-ing.yaml
ingress.networking.k8s.io/grafana created

校验测试

查看pod/svc信息

[root@master1 prometheus-k8s]# kubectl get pods -A -o wide

[root@master1 prometheus-k8s]# kubectl get svc -A

查看prometheus

访问http://192.168.136.165:30003，这是Prometheus的页面，依次点击Status>Targets可以看到已经成功连接到k8s的apiserver

查看node exporter

访问http://192.168.136.161:31672/metrics，这是node-exporter采集的数据。

查看grafana

访问http://192.168.136.165:30950，这是grafana的页面，账户、密码都是admin。

创建Dashboard

add data souce