k8s之Prometheus-v2.35.0部署

Prometheus-v2.35.0

主机	IP
k8s-master-1(/promethues 提供持久化存储)	192.168.0.10
k8s-node-1	192.168.0.11

Prometheus-server配置文件：https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus-kubernetes.yml

Prometheus-RBAC

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources: ["nodes","nodes/proxy","services","endpoints","pods"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["extensions"]
  resources: ["ingresses"]
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: kube-prometheus
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: kube-prometheus

Prometheus-Configmap

[root@k8s-master-1 prometheus-server]# cat prometheus-config.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: kube-prometheus
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s 
      scrape_timeout: 10s
      evaluation_interval: 1m
    scrape_configs:
    - job_name: 'prometheus-server'
      static_configs:
      - targets: ['localhost:9090']
      relabel_configs:
      - source_labels: [instance]             # 将instance=localhost:9090 -> instance=prometheus-server:9090
        regex: '(.*)'
        replacement: 'prometheus-server:9090'
        target_label: instance
        action: replace
    - job_name: 'kubernetes-node'
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - source_labels: [__address__]          # 将__address__=192.168.0.10:10250 -> 192.168.0.10:9100,node_exporter默认是9100，且我已事先与宿主机共享网络空间了
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - action: labelmap                 
        regex: '__meta__kubernetes_node_label_(.+)'  # 保留__meta__kubernetes_node_label_标签后面的值作为新标签
    - job_name: 'kubernetes-apiserver'
      kubernetes_sd_configs:
      - role: endpoints                              
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels:
          [
            __meta_kubernetes_namespace,
            __meta_kubernetes_service_name,
            __meta_kubernetes_endpoint_port_name,
          ]
        action: keep
        regex: default;kubernetes;https
    - job_name: "kubernetes-cadvisor"
      scheme: https
      metrics_path: /metrics/cadvisor
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      authorization:
        credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)  # 保留匹配到的具有__meta_kubernetes_node_label的标签
      - target_label: __address__                 # 获取到的地址:__address__="192.168.0.11:10250"
        replacement: kubernetes.default.svc:443   # 把获取到的地址替换成新的地址kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__             # 获取 __metrics_ 对应的值
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor # 把metrics替换成新的值,新的url就是: https://kubernetes.default.svc/api/v1/nodes/k8s-master-1/proxy/metrics/cadvisor
    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scraped] # 仅抓取到的具有"prometheus.io/scrape: true"的annotation的端点
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] # 应用中自定义暴露的指标，也许你暴露的 API 接口不是 /metrics 这个路径，那么你可以在这个POD 对应的 service 中做一个 "prometheus.io/path = /mymetrics"声明
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)     # ; 对应 source_labels里面的
        replacement: $1:$2                # 暴露自定义的应用的端口，就是把地址和你在 service 中定义的 "prom etheus.io/port=<port>" 声明做一个拼接然后赋值给 __ address__，这样 prometheus 就能获取自定义应用的端口，然后通过这个端口再结合 __metrics_ 来获 取 指标，如果 __metrics_ 值不是默认的 /metrics 那么就要使用上面的标签替换来获取真正暴露的具体路径
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_service_name 

Prometheus-Deployment

[root@k8s-master-1 prometheus-server]# cat prometheus-deployment.yaml 
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: kube-prometheus
  labels:
    name: prometheus-server
spec:
  ports:
  - name: prometheus
    protocol: TCP
    port: 9090
    targetPort: 9090
    nodePort: 40000
  selector:
    name: prometheus-server
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: kube-prometheus
  labels:
    name: prometheus-server
spec:
  replicas: 1
  selector:
    matchLabels:
      name: prometheus-server
  template:
    metadata:
      labels:
        name: prometheus-server
      annotations:
#        prometheus.io/scrape: 'true'    # The default configuration will scrape all pods and, if set to false, this annotation will exclude the pod from the scraping process.
#        prometheus.io/path: '/metrics'  # If the metrics path is not /metrics, define it with this annotation.
#        prometheus.io/port: '9090'      # Scrape the pod on the indicated port instead of the pod’s declared ports (default is a port-free target if none are declared)
    spec:
      nodeName: k8s-master-1      # 设置在k8s-master-1上运行
      tolerations:                # 设置能容忍在master节点运行
      - key: "node-role.kubernetes.io/master"
        operator: "Exists"
        effect: "NoSchedule"
      serviceAccountName: prometheus
      containers:
      - name: prometheus
        image: prom/prometheus:v2.35.0
        imagePullPolicy: IfNotPresent
        command:
          - prometheus
          - --config.file=/etc/prometheus/prometheus.yml
          - --storage.tsdb.path=/prometheus       # 旧数据存储目录
          - --storage.tsdb.retention=720h         # 何时删除旧数据，默认为15天。
          - --web.enable-lifecycle                # 开启热加载
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - name: prometheus-config
          mountPath: /etc/prometheus/prometheus.yml
          subPath: prometheus.yml
        - name: prometheus-storage-volume
          mountPath: /prometheus/
        readinessProbe:
          httpGet:
            path: /-/ready
            port: 9090
          failureThreshold: 3
          successThreshold: 1
          periodSeconds: 5
          initialDelaySeconds: 5
          timeoutSeconds: 5
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: 9090
          failureThreshold: 3
          successThreshold: 1
          periodSeconds: 5
          initialDelaySeconds: 15
          timeoutSeconds: 5
      volumes:
        - name: prometheus-config
          configMap:
            name: prometheus-config
            items:
              - key: prometheus.yml
                path: prometheus.yml
                mode: 0644
        - name: prometheus-storage-volume
          hostPath:
           path: /prometheus
           type: Directory

部署后效果如下：