Kube-prometheus部署Prometheus
下载 kube-prometheus
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.10.0.tar.gz
cd /data/bigdata/kube-prometheus-0.10.0/manifests
grep -rn 'image: '
docker tag quay.io/prometheus/alertmanager:v0.23.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/alertmanager:v0.23.0
docker tag quay.io/prometheus/blackbox-exporter:v0.19.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/blackbox-exporter:v0.19.0
docker tag quay.io/brancz/kube-rbac-proxy:v0.11.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-rbac-proxy:v0.11.0
docker tag quay.io/prometheus/node-exporter:v1.3.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/node-exporter:v1.3.1
docker tag quay.io/prometheus/prometheus:v2.32.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus:v2.32.1
docker tag quay.io/prometheus-operator/prometheus-config-reloader:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-config-reloader:v0.53.1
docker tag quay.io/prometheus-operator/prometheus-operator:v0.53.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-operator:v0.53.1
docker tag bitnami/kube-state-metrics:2.3.0 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/kube-state-metrics:2.3.0
docker tag willdockerhub/prometheus-adapter:v0.9.1 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/prometheus-adapter:v0.9.1
docker tag grafana/grafana:8.3.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/grafana:8.3.3
docker tag prom/pushgateway:v1.4.3 bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
docker push bigdata-uat-k8s-harbor.ky-tech.com.cn/bigdata/prometheus/pushgateway:v1.4.3
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' alertmanager-alertmanager.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' blackboxExporter-deployment.yaml
sed -i 's/quay.io\/brancz/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' kubeStateMetrics-deployment.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' nodeExporter-daemonset.yaml
sed -i 's/quay.io/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata/g' prometheus-prometheus.yaml
sed -i 's/quay.io\/prometheus-operator/bigdata-uat-k8s-harbor.ky-tech.com.cn\/bigdata\/prometheus/g' prometheusOperator-deployment.yaml
修改类型为 NodePort
cd /data/bigdata/kube- prometheus- 0.10.0/manifests
vim prometheus- service.yaml
spec :
type : NodePort
ports :
- name : web
port : 9090
targetPort : web
nodePort : 30090
- name : reloader- web
port : 8080
targetPort : reloader- web
vim alertmanager- service.yaml
spec :
type : NodePort
ports :
- name : web
port : 9093
targetPort : web
nodePort : 30093
- name : reloader- web
port : 8080
targetPort : reloader- web
vim grafana- service.yaml
spec :
type : NodePort
ports :
- name : http
port : 3000
targetPort : http
nodePort : 30300
kube-prometheus 集成 pushgateway
apiVersion : apps/v1
kind : Deployment
metadata :
labels :
app.kubernetes.io/name : pushgateway
name : pushgateway
namespace : monitoring
spec :
replicas : 1
selector :
matchLabels :
app.kubernetes.io/name : pushgateway
template :
metadata :
labels :
app.kubernetes.io/name : pushgateway
spec :
nodeSelector :
kubernetes.io/os : linux
containers :
- image : bigdata- uat- k8s- harbor.ky- tech.com.cn/bigdata/prometheus/pushgateway: v1.4.3
name : pushgateway
ports :
- containerPort : 9091
name : http
resources :
limits :
cpu : 200m
memory : 512Mi
requests :
cpu : 50m
memory : 128Mi
restartPolicy : Always
---
apiVersion : v1
kind : Service
metadata :
labels :
app.kubernetes.io/name : pushgateway
name : pushgateway
namespace : monitoring
spec :
type : NodePort
ports :
- name : http
port : 9091
targetPort : http
nodePort : 30391
selector :
app.kubernetes.io/name : pushgateway
---
apiVersion : monitoring.coreos.com/v1
kind : ServiceMonitor
metadata :
labels :
app.kubernetes.io/name : pushgateway
name : pushgateway
namespace : monitoring
spec :
endpoints :
- interval : 30s
path : /metrics
port : http
scheme : http
selector :
matchLabels :
app.kubernetes.io/name : pushgateway
修改grafana配置
vim grafana-config.yaml
grafana.ini: |
[ security]
allow_embedding = true
[ date_formats]
default_timezone = UTC
[ auth.anonymous]
enabled = true
[ database]
type = mysql
host = mysql-master.mysql:3306
name = grafana
user = grafana
password = YQZi5UUw825h
type: Opaque
Prometheus 持久化存储
vim sc.yaml
kind : StorageClass
apiVersion : storage.k8s.io/v1
metadata :
name : prometheus- local- storage
provisioner : kubernetes.io/no- provisioner
volumeBindingMode : WaitForFirstConsumer
reclaimPolicy : Retain
vim pv.yaml
apiVersion : v1
kind : PersistentVolume
metadata :
name : prometheus- pv- 1
spec :
storageClassName : prometheus- local- storage
capacity :
storage : 10Gi
accessModes :
- ReadWriteOnce
local :
path : /data/zxl/prometheus1/
nodeAffinity :
required :
nodeSelectorTerms :
- matchExpressions :
- key : kubernetes.io/hostname
operator : In
values :
- szzb- bg- dev- etl- 4
---
apiVersion : v1
kind : PersistentVolume
metadata :
name : prometheus- pv- 2
spec :
storageClassName : prometheus- local- storage
capacity :
storage : 10Gi
accessModes :
- ReadWriteOnce
local :
path : /data/zxl/prometheus2/
nodeAffinity :
required :
nodeSelectorTerms :
- matchExpressions :
- key : kubernetes.io/hostname
operator : In
values :
- szzb- bg- dev- etl- 5
vim prometheus- prometheus.yaml
retention : 15d
storage :
volumeClaimTemplate :
spec :
storageClassName : prometheus- local- storage
resources :
requests :
storage : 10Gi
配置Ingress
vim ingress.yaml
apiVersion : networking.k8s.io/v1
kind : Ingress
metadata :
name : prometheus- ingress
namespace : monitoring
annotations :
kubernetes.io/ingress.class : "nginx"
prometheus.io/http_probe : "true"
spec :
rules :
- host : alert.k8s.com
http :
paths :
- path : /
pathType : Prefix
backend :
service :
name : alertmanager- main
port :
number : 9093
- host : bigdata- uat- k8s- grafana.ky- tech.com.cn
http :
paths :
- path : /
pathType : Prefix
backend :
service :
name : grafana
port :
number : 3000
tls :
- hosts :
- bigdata- uat- k8s- grafana.ky- tech.com.cn
secretName : grafana- secret
- host : bigdata- uat- k8s- prometheus.ky- tech.com.cn
http :
paths :
- path : /
pathType : Prefix
backend :
service :
name : prometheus- k8s
port :
number : 9090
tls :
- hosts :
- bigdata- uat- k8s- prometheus.ky- tech.com.cn
secretName : prometheus- secret
kubectl create secret tls grafana-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
kubectl create secret tls prometheus-secret --key /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn.key --cert /data/k8s-install/ssl/ky-tech.com.cn_nginx/ky-tech.com.cn_bundle.crt -n monitoring
kubectl apply -rf .
部署
kubectl apply --server-side -f manifests/setup
kubectl wait \
--for condition = Established \
--all CustomResourceDefinition \
--namespace = monitoring
kubectl apply -f manifests/
kubectl delete --ignore-not-found= true -f manifests/ -f manifests/setup
监控k8s集群外节点
---
apiVersion : monitoring.coreos.com/v1
kind : ServiceMonitor
metadata :
labels :
app.kubernetes.io/component : exporter
app.kubernetes.io/name : nginx- exporter
app.kubernetes.io/app : nginx
name : nginx
namespace : monitoring
spec :
endpoints :
- interval : 30s
port : metrcis
selector :
matchLabels :
app.kubernetes.io/component : exporter
app.kubernetes.io/name : nginx- exporter
app.kubernetes.io/app : nginx
namespaceSelector :
matchNames :
- monitoring
---
apiVersion : v1
kind : Service
metadata :
labels :
app : nginx- exporter
app.kubernetes.io/component : exporter
app.kubernetes.io/name : nginx- exporter
app.kubernetes.io/app : nginx
name : nginx
namespace : monitoring
spec :
type : ClusterIP
clusterIP : None
ports :
- name : metrcis
port : 9113
targetPort : 9113
protocol : TCP
---
apiVersion : v1
kind : Endpoints
metadata :
labels :
app.kubernetes.io/component : exporter
app.kubernetes.io/name : nginx- exporter
app.kubernetes.io/app : nginx
name : nginx
namespace : monitoring
subsets :
- addresses :
- ip : 192.168.26.11
- ip : 192.168.26.13
ports :
- name : metrcis
port : 9113
protocol : TCP
Kube-prometheus 监控 K8s 集群外服务的两种方式