二进制k8s集群安装prometheus-operator
环境准备
##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.54 nfs
## keepalived的vip
192.168.48.66
IP | Hostname | CPU | Memory |
---|---|---|---|
192.168.48.101 | master01 | 2 | 4G |
192.168.48.102 | master02 | 2 | 4G |
192.168.48.103 | master03 | 2 | 4G |
192.168.48.201 | node01 | 2 | 4G |
192.168.48.202 | node02 | 2 | 4G |
192.168.48.54 | nfs | 2 | 4G |
软件 | 版本 |
---|---|
kubernetes | 1.15.2 |
docker-ce | 19.03 |
calico | 3.8 |
etcd | 3.3.13 |
CNI | 0.8.1 |
coredns | 1.4.0 |
metrics-server | 0.3.3 |
ingress-controller | 0.25.0 |
dashboard | 1.10.1 |
Weave Scope | 1.11.4 |
nfs | v4 |
helm | 2.14.3 |
harbor | 1.1.1 |
安装
下载部署文件
https://github.com/coreos/kube-prometheus.git
下载地址
链接: https://pan.baidu.com/s/1uA4Lf3ZUfg0b1YLpoSRGHw 提取码: pw58
unzip kube-prometheus-0.2.0.zip
mkdir prometheus
cp kube-prometheus-0.2.0/manifests/* prometheus/
cd prometheus/
mkdir -p operator node-exporter alertmanager grafana kube-state-metrics prometheus serviceMonitor adapter
mv *-serviceMonitor* serviceMonitor/
mv 0prometheus-operator* operator/
mv grafana-* grafana/
mv kube-state-metrics-* kube-state-metrics/
mv alertmanager-* alertmanager/
mv node-exporter-* node-exporter/
mv prometheus-adapter* adapter/
mv prometheus-* prometheus/
[root@master01 prometheus]# pwd
/root/prometheus
[root@master01 prometheus]# ll
total 20
-rw-r--r-- 1 root root 60 Aug 18 21:01 00namespace-namespace.yaml
drwxr-xr-x 2 root root 4096 Aug 18 21:01 adapter
drwxr-xr-x 2 root root 149 Aug 18 21:01 alertmanager
drwxr-xr-x 2 root root 219 Aug 18 21:01 grafana
drwxr-xr-x 2 root root 305 Aug 18 21:01 kube-state-metrics
drwxr-xr-x 2 root root 200 Aug 18 21:01 node-exporter
drwxr-xr-x 2 root root 4096 Aug 18 21:01 operator
drwxr-xr-x 2 root root 4096 Aug 18 21:01 prometheus
drwxr-xr-x 2 root root 4096 Aug 18 21:01 serviceMonitor
查看node的label
[root@master01 prometheus]# curl -s https://zhangguanzhang.github.io/bash/label.sh | bash
Node Label
master01
app ingress
beta.kubernetes.io/arch amd64
beta.kubernetes.io/os linux
kubernetes.io/arch amd64
kubernetes.io/hostname master01
kubernetes.io/os linux
node-role.kubernetes.io/master
master02
beta.kubernetes.io/arch amd64
beta.kubernetes.io/os linux
kubernetes.io/arch amd64
kubernetes.io/hostname master02
kubernetes.io/os linux
node-role.kubernetes.io/master
master03
beta.kubernetes.io/arch amd64
beta.kubernetes.io/os linux
kubernetes.io/arch amd64
kubernetes.io/hostname master03
kubernetes.io/os linux
node-role.kubernetes.io/master
node01
beta.kubernetes.io/arch amd64
beta.kubernetes.io/os linux
kubernetes.io/arch amd64
kubernetes.io/hostname node01
kubernetes.io/os linux
node-role.kubernetes.io/node
node02
beta.kubernetes.io/arch amd64
beta.kubernetes.io/os linux
kubernetes.io/arch amd64
kubernetes.io/hostname node02
kubernetes.io/os linux
node-role.kubernetes.io/node
查看是否匹配prometheus里的label
[root@master01 prometheus]# grep -A1 nodeSelector alertmanager/alertmanager-alertmanager.yaml
nodeSelector:
kubernetes.io/os: linux
镜像下载
quay.io/prometheus/node-exporter:v0.18.1
quay.io/prometheus/alertmanager:v0.18.0
quay.io/prometheus/prometheus:v2.11.0
quay.io/coreos/configmap-reload:v0.0.1
quay.io/coreos/prometheus-config-reloader:v0.33.0
quay.io/coreos/prometheus-operator:v0.33.0
quay.io/coreos/kube-rbac-proxy:v0.4.1
quay.io/coreos/kube-state-metrics:v1.7.2
quay.io/coreos/k8s-prometheus-adapter-amd64:v0.4.1
k8s.gcr.io/addon-resizer:1.8.4
grafana/grafana:6.2.2
链接: https://pan.baidu.com/s/19tUbs8bBQYDrLeHNcKXv-Q 提取码: qs7t
docker load -i prom.tar.gz
部署namespace
[root@master01 prometheus]# kubectl apply -f 00namespace-namespace.yaml
namespace/monitoring created
部署operator
[root@master01 prometheus]# kubectl apply -f operator/
customresourcedefinition.apiextensions.k8s.io/alertmanagers.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/podmonitors.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheuses.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheusrules.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/servicemonitors.monitoring.coreos.com created
clusterrole.rbac.authorization.k8s.io/prometheus-operator created
clusterrolebinding.rbac.authorization.k8s.io/prometheus-operator created
deployment.apps/prometheus-operator created
service/prometheus-operator created
serviceaccount/prometheus-operator created
[root@master01 prometheus]# kubectl -n monitoring get pod
NAME READY STATUS RESTARTS AGE
prometheus-operator-7447bf4dcb-qw4jn 1/1 Running 0 27s
部署整套CRD
kubectl apply -f adapter/
kubectl apply -f alertmanager/
kubectl apply -f node-exporter/
kubectl apply -f kube-state-metrics/
kubectl apply -f grafana/
kubectl apply -f prometheus/
kubectl apply -f serviceMonitor/
查看整体部署情况
[root@master01 prometheus]# kubectl -n monitoring get all
NAME READY STATUS RESTARTS AGE
pod/alertmanager-main-0 2/2 Running 0 88s
pod/alertmanager-main-1 2/2 Running 0 63s
pod/alertmanager-main-2 2/2 Running 0 47s
pod/grafana-7dc5f8f9f6-n7vws 1/1 Running 0 73s
pod/kube-state-metrics-59c47cfcfb-q9255 4/4 Running 0 62s
pod/node-exporter-2psjp 2/2 Running 0 87s
pod/node-exporter-7tcsb 2/2 Running 0 87s
pod/node-exporter-hm8g8 2/2 Running 0 87s
pod/node-exporter-mccdw 2/2 Running 0 87s
pod/node-exporter-xtxc2 2/2 Running 0 87s
pod/prometheus-adapter-668748ddbd-nwmxl 1/1 Running 0 89s
pod/prometheus-k8s-0 3/3 Running 0 72s
pod/prometheus-k8s-1 3/3 Running 1 72s
pod/prometheus-operator-7447bf4dcb-qw4jn 1/1 Running 0 2m35s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/alertmanager-main ClusterIP 10.103.101.118 <none> 9093/TCP 89s
service/alertmanager-operated ClusterIP None <none> 9093/TCP,6783/TCP 89s
service/grafana ClusterIP 10.102.56.50 <none> 3000/TCP 74s
service/kube-state-metrics ClusterIP None <none> 8443/TCP,9443/TCP 87s
service/node-exporter ClusterIP None <none> 9100/TCP 88s
service/prometheus-adapter ClusterIP 10.97.209.217 <none> 443/TCP 90s
service/prometheus-k8s ClusterIP 10.107.103.78 <none> 9090/TCP 72s
service/prometheus-operated ClusterIP None <none> 9090/TCP 72s
service/prometheus-operator ClusterIP None <none> 8080/TCP 2m36s
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/node-exporter 5 5 5 5 5 kubernetes.io/os=linux 88s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/grafana 1/1 1 1 74s
deployment.apps/kube-state-metrics 1/1 1 1 88s
deployment.apps/prometheus-adapter 1/1 1 1 90s
deployment.apps/prometheus-operator 1/1 1 1 2m36s
NAME DESIRED CURRENT READY AGE
replicaset.apps/grafana-7dc5f8f9f6 1 1 1 74s
replicaset.apps/kube-state-metrics-59c47cfcfb 1 1 1 62s
replicaset.apps/kube-state-metrics-77467ddf9b 0 0 0 88s
replicaset.apps/prometheus-adapter-668748ddbd 1 1 1 90s
replicaset.apps/prometheus-operator-7447bf4dcb 1 1 1 2m36s
NAME READY AGE
statefulset.apps/alertmanager-main 3/3 89s
statefulset.apps/prometheus-k8s 2/2 72s
创建ingress
[root@master01 prometheus]# kubectl get svc -n monitoring
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
alertmanager-main ClusterIP 10.103.101.118 <none> 9093/TCP 6m22s
alertmanager-operated ClusterIP None <none> 9093/TCP,6783/TCP 6m22s
grafana ClusterIP 10.102.56.50 <none> 3000/TCP 6m7s
kube-state-metrics ClusterIP None <none> 8443/TCP,9443/TCP 6m20s
node-exporter ClusterIP None <none> 9100/TCP 6m21s
prometheus-adapter ClusterIP 10.97.209.217 <none> 443/TCP 6m23s
prometheus-k8s ClusterIP 10.107.103.78 <none> 9090/TCP 6m5s
prometheus-operated ClusterIP None <none> 9090/TCP 6m5s
prometheus-operator ClusterIP None <none> 8080/TCP 7m29s
vim prometheus-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: prometheus.com
namespace: monitoring
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: grafana.tk8s.com
http:
paths:
- backend:
serviceName: grafana
servicePort: 3000
- host: prometheus.tk8s.com
http:
paths:
- backend:
serviceName: prometheus-k8s
servicePort: 9090
- host: alertmanager.tk8s.com
http:
paths:
- backend:
serviceName: alertmanager-main
servicePort: 9093
[root@master01 prometheus]# kubectl apply -f prometheus-ingress.yaml
ingress.extensions/prometheus.com created
访问prometheus
所有master修改systemd配置文件,修改bind
vim /usr/lib/systemd/system/kube-controller-manager.service
....
--bind-address=0.0.0.0 \
systemctl daemon-reload
systemctl restart kube-controller-manager.service
vim /usr/lib/systemd/system/kube-scheduler.service
....
--address=0.0.0.0 \
systemctl daemon-reload
systemctl restart systemctl restart kube-scheduler.service
创建kube-controller-manager和kube-scheduler的svc
vim prom-svc.yaml
apiVersion: v1
kind: Service
metadata:
namespace: kube-system
name: kube-controller-manager
labels:
k8s-app: kube-controller-manager
spec:
type: ClusterIP
clusterIP: None
ports:
- name: http-metrics
port: 10252
targetPort: 10252
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
namespace: kube-system
name: kube-scheduler
labels:
k8s-app: kube-scheduler
spec:
type: ClusterIP
clusterIP: None
ports:
- name: http-metrics
port: 10251
targetPort: 10251
protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
labels:
k8s-app: kube-controller-manager
name: kube-controller-manager
namespace: kube-system
subsets:
- addresses:
- ip: 192.168.48.101
- ip: 192.168.48.102
- ip: 192.168.48.103
ports:
- name: http-metrics
port: 10252
protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
labels:
k8s-app: kube-scheduler
name: kube-scheduler
namespace: kube-system
subsets:
- addresses:
- ip: 192.168.48.101
- ip: 192.168.48.102
- ip: 192.168.48.103
ports:
- name: http-metrics
port: 10251
protocol: TCP
[root@master01 prometheus]# kubectl apply -f prom-svc.yaml
访问grafana
grafana初始用户名和密码都是是admin