prometheus监控

prometheus监控

一、prometheus监控部署

1.1创建授权

kubectl create ns monitoring
kubectl create serviceaccount monitor -n monitoring 
kubectl create clusterrolebinding monitor-clusterrolebinding -n monitoring --clusterrole=cluster-admin --serviceaccount=monitoring:monitor

1.2 创建数据共享目录

prometheus调度到哪个节点上就在哪个节点上创建数据共享目录
mkdir -p /apps/sharedstorage/jtproduction-hlw/prometheus/conf/fs.d
mkdir -p /apps/sharedstorage/jtproduction-hlw/prometheus/data
chmod 777 /apps/sharedstorage/jtproduction-hlw/prometheus/conf/fs.d
chmod 777 /apps/sharedstorage/jtproduction-hlw/prometheus/data

1.3 创建secret

apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
  name: token-secret
  namespace: monitoring
  annotations:
    kubernetes.io/service-account.name: monitor

1.4

2 # k8s.token  需要删除多余内容
3 kubectl describe secrets -n monitoring token-secret >> k8s.token
4
5 # ca.crt 
6 kubectl get secrets -n monitoring token-secret -oyaml | grep "ca.crt:"| awk '{print $2}' | base64 -d >>ca.crt

1.5 创建configmap.yaml文件（只需改ip）

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    app: prometheus
  name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 1m
    scrape_configs:
    - job_name: 'kubernetes-node'
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)

    - job_name: 'kubernetes-gpu'
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels: [__address__]
        action: keep
        regex: '(.*):9400'
      - source_labels: [__meta_kubernetes_pod_node_name]
        action: replace
        target_label: node
      - source_labels: [__meta_kubernetes_pod_host_ip]
        action: replace
        target_label: node_ip
        
    - job_name: kubernetes-pods
      bearer_token_file: /etc/prometheus/fs.d/k8s.token
      scheme: https
      tls_config:
        ca_file: /etc/prometheus/fs.d/ca.crt
        insecure_skip_verify: true
      kubernetes_sd_configs:
      - api_server: https://10.246.29.43:6443        ####修改
        role: pod
        bearer_token_file: /etc/prometheus/fs.d/k8s.token
        tls_config:
          ca_file: /etc/prometheus/fs.d/ca.crt
          insecure_skip_verify: true
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
      - regex: '([^;]+);([^;]+);([^;]+);([^;]+)'
        source_labels:
        - __meta_kubernetes_namespace
        - __meta_kubernetes_pod_name
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        - __meta_kubernetes_pod_annotation_prometheus_io_path
        target_label: __metrics_path__
        replacement: '/api/v1/namespaces/${1}/pods/http:${2}:${3}/proxy${4}'
      - target_label: __address__
        replacement: '10.246.29.43:6443'            ####修改
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: kubernetes_pod_name
      - action: drop
        regex: 'Pending|Succeeded|Failed'
        source_labels:
        - __meta_kubernetes_pod_phase     
  config.yml: |
    basic_auth_users:
      admin: $2y$12$oeXC3x36W3rHAluW.ocSIulKBEzObsqACk0.8Pv/hKp4lotL0V/MS

1.6创建prometheus.yaml文件

apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: monitoring
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      component: server
  template:
    metadata:
      labels:
        app: prometheus
        component: server
      annotations:
        prometheus.io/scrape: 'false'
    spec:
      serviceAccountName: monitor
      containers:
      - name: prometheus
        image: mirror.hcjt.k8s.com:8989/rancher/prometheus:v2.42         ####修改
        imagePullPolicy: IfNotPresent
        command:
          - prometheus
          - --config.file=/etc/prometheus/prometheus.yml
          - --storage.tsdb.path=/prometheus
          - --storage.tsdb.retention=35d
          - --web.enable-lifecycle
          - --web.config.file=/etc/prometheus/config.yml
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - name: prometheus-config
          mountPath: /etc/prometheus/prometheus.yml
          subPath: prometheus.yml
        - name: prometheus-config
          mountPath: /etc/prometheus/config.yml
          subPath: config.yml
        - name: prometheus-storage-volume
          mountPath: /prometheus/
        - name: prometheus-config-volume
          mountPath: /etc/prometheus/fs.d/
        - name: prometheus-time
          mountPath: /etc/localtime
      volumes:
      - name: prometheus-config
        configMap:
          name: prometheus-config
          items:
          - key: prometheus.yml
            path: prometheus.yml
            mode: 0644
          - key: config.yml
            path: config.yml
            mode: 0644
      - name: prometheus-storage-volume
        hostPath:
          path: /apps/sharedstorage/jtproduction-hlw/prometheus/data           ####修改
          type: DirectoryOrCreate
      - name: prometheus-config-volume
        hostPath:
          path: /apps/sharedstorage/jtproduction-hlw/prometheus/conf/fs.d      ####修改
          type: DirectoryOrCreate
      - name: prometheus-time
        hostPath:
          path: /etc/localtime
          type: ""

1.7创建service对外暴露端口

apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitoring
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
    - port: 9090
      targetPort: 9090
      protocol: TCP
      nodePort: 30042
  selector:
    app: prometheus
    component: server