promethues部署elasticsearch_exporter,mysql-exporter并添加报警配置

1. 部署elasticsearch_exporter

#cat expe-es.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: elaesticsearch
  namespace: prometheus
spec:
  selector:
    matchLabels:
      app: elaesticsearch
  template:
    metadata:
      annotations:  #三个注解必须要带上，不然promethues不能匹配到
        prometheus.io/scrape: 'true'  
        prometheus.io/port: '9114'
        prometheus.io/path: 'metrics'
      labels:
        app: elaesticsearch
      name: elaesticsearch
    spec:
      containers:
      - name: elaesticsearch
        image: justwatch/elasticsearch_exporter:1.1.0
        command:
        - "/bin/elasticsearch_exporter"
        - "--es.uri=http://192.168.10.100:9200"   #es的地址
        ports:
        - containerPort: 9114
          name: scrape

1.1 部署mysql-exporter

#cat  mysql-exporter.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: mysqld
  namespace: prometheus
spec:
  template:
    metadata:
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/port: '9104'
        prometheus.io/path: 'metrics'
      labels:
        app: mysqld
      namespace: prometheus
      name: mysqld
    spec:
      containers:
      - name: mysqld
        image: prom/mysqld-exporter
        args:
          - --collect.info_schema.innodb_metrics
          - --collect.info_schema.processlist
          - --collect.info_schema.innodb_cmp
          - --collect.info_schema.innodb_cmpmem
        env:
        - name: DATA_SOURCE_NAME
          value: user:password@(192.168.1.123:3306)/
        ports:
        - containerPort: 9104
          name: scrape

参考:https://cloud.tencent.com/document/product/1416/56035

2. 启动

kubectl apply -f expe-es.yaml

3. 查看promethues配置

cat  prometheus-conf.yaml
...

      - job_name: 'kubernetes-pods'
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: kubernetes_pod_name
        - source_labels: [__meta_pod_template_generation]
          action: replace
          target_label: __metrics_path__
          regex: (.+)

...

4. 刷新配置，重启promethues

kubectl apply -f prometheus-conf.yaml
kubectl rollout restart deployment   prometheus -n prometheus

5. 查看promethues
   
6. 配置告警
   elasticsearch_cluster_health_number_of_data_nodes node节点的数量
   elasticsearch_node_stats_up es状态是否正常
   

cat   prometheus-rules.yaml
...

      - alert: elas-stop
        annotations:
          description:  elas 9200 stop
        expr: elasticsearch_node_stats_up{app="elaesticsearch"} !=1
        for: 2m
        labels:
          severity: critical
          service:  gaoji

...

7. 刷新配置

kubectl apply -f prometheus-rules.yaml

8. 关闭es测试一下
   

#申明这个promethues是手动部署的，可以直接去我的资源下载，使用前先看说明直接可以用。
https://download.csdn.net/download/weixin_43606975/85824711?spm=1001.2014.3001.5503