14. kubernetes——promethues监控

一、Promethues监控系统架构

• Prometheus Server
  Prometheus 生态最重要的组件，主要用于抓取和存储时间 序列数据，同时提供数据的查询和告警策略的配置管理；

• Alertmanager
  Prometheus 生态用于告警的组件，Prometheus Server 会将告警发送给 Alertmanager，Alertmanager 根据路由配置，将告警信息发送给指定的人或组。Alertmanager 支持邮件、Webhook、微信、钉钉、短信等媒介进行告 警通知

• Grafana
  用于展示数据，便于数据的查询和观测；

• Push Gateway
  Prometheus 本身是通过 Pull 的方式拉取数据，但是有些监控数 据可能是短期的，如果没有采集数据可能会出现丢失。Push Gateway 可以用来 解决此类问题，它可以用来接收数据，也就是客户端可以通过 Push 的方式将数据推送到 Push Gateway，之后 Prometheus 可以通过 Pull 拉取该数据；

• Exporter
  主要用来采集监控数据，比如主机的监控数据可以通过 node_exporter 采集，MySQL 的监控数据可以通过 mysql_exporter 采集，之后 Exporter 暴露一 个接口，比如/metrics，Prometheus 可以通过该接口采集到数据；

• PromQL
  PromQL 其实不算 Prometheus 的组件，它是用来查询数据的一种语法，比如查询数据库的数据，可以通过SQL语句，查询Loki的数据，可以通过LogQL，查询 Prometheus 数据的叫做 PromQL；

• Service Discovery
  用来发现监控目标的自动发现，常用的有基于 Kubernetes、 Consul、Eureka、文件的自动发现等。

二、部署promethues server

1、创建promethues configmap配置

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s
    scrape_configs:
      - job_name: 'prometheus'
        static_configs:
        - targets: ['localhost:9090']

[root@k8s-master promethues]# kubectl create -f promethues-config.yaml 
configmap/prometheus-config created

2、创建promethues server

apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - name: prometheus
        image: prom/prometheus:v2.2.1
        imagePullPolicy: IfNotPresent
        command:
        - "/bin/prometheus"
        args:
        - "--config.file=/etc/prometheus/prometheus.yml"
        ports:
        - containerPort: 9090
        volumeMounts:
        - mountPath: "/etc/prometheus"
          name: prometheus-config-volume
      volumes:
      - name: prometheus-config-volume
        configMap:
          name: prometheus-config
---
apiVersion: v1
kind: "Service"
metadata:
  name: prometheus
spec:
  ports:
  - name: prometheus
    protocol: TCP
    port: 9090
    targetPort: 9090
  selector:
    app: prometheus
  type: NodePort

[root@k8s-master promethues]# kubectl create -f promethues.yaml 
deployment.apps/prometheus created
service/prometheus created

[root@k8s-master promethues]# kubectl get pod 
NAME                          READY   STATUS    RESTARTS   AGE
prometheus-5fb5c655dd-pcqzk   1/1     Running   0          5s

[root@k8s-master promethues]# kubectl get svc
NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
kubernetes   ClusterIP   10.96.0.1      <none>        443/TCP          127d
prometheus   NodePort    10.96.45.187   <none>        9090:32610/TCP   12s

3、创建授权用户

用户授权，基于RBAC(基于角色的访问控制)

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/proxy
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: default

[root@k8s-master promethues]# kubectl create -f rbac.yaml 
clusterrole.rbac.authorization.k8s.io/prometheus created
serviceaccount/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created

4、修改promethues server的配置，指定上述的授权用户

..............................................
    spec:
      serviceAccountName: prometheus     //添加关联用户
      containers:
  ..............................................

[root@k8s-master promethues]# kubectl apply -f promethues.yaml 
[root@k8s-master promethues]# kubectl get pod
NAME                          READY   STATUS    RESTARTS   AGE
prometheus-5fb5c655dd-pcqzk   1/1     Running   0          17m

添加serviceAccount的pod会自动将访问API的CA证书及用户的令牌挂载到/var/run/secrets/kubernetes.io/serviceaccount/目录下

[root@k8s-master promethues]# kubectl exec -ti prometheus-5fb5c655dd-pcqzk ls /var/run/secrets/kubernetes.io/serviceaccount/ 
ca.crt     namespace  token
[root@k8s-master promethues]# 

三、通过node_exporter监控主机状态

1、使用daemonset类型pod部署node_exporter

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/port: '9100'
        prometheus.io/path: 'metrics'
      labels:
        app: node-exporter
      name: node-exporter
    spec:
      containers:
      - image: prom/node-exporter
        imagePullPolicy: IfNotPresent
        name: node-exporter
        ports:
        - containerPort: 9100
          hostPort: 9100
          name: scrape
      hostNetwork: true
      hostPID: true

[root@k8s-master promethues]# kubectl create -f node_exporter.yaml 
daemonset.apps/node-exporter created
[root@k8s-master promethues]# 
[root@k8s-master promethues]# kubectl get pod -o wide
NAME                          READY   STATUS    RESTARTS   AGE    IP               NODE                   NOMINATED NODE   READINESS GATES
node-exporter-kf4cs           1/1     Running   0          9s     192.168.140.12   k8s-node02.linux.com   <none>           <none>
node-exporter-q68q9           1/1     Running   0          9s     192.168.140.11   k8s-node01.linux.com   <none>           <none>

2、测试获取数据

[root@k8s-master promethues]# curl http://192.168.140.11:9100/metrics 

3、修改promethues配置文件，添加监控任务

[root@k8s-master promethues]# vim promethues-config.yaml 

      - job_name: 'node_11'
        static_configs:
        - targets: ["192.168.140.11:9100"]

      - job_name: 'node_12'
        static_configs:
        - targets: ["192.168.140.12:9100"]

[root@k8s-master promethues]# kubectl apply -f promethues-config.yaml

[root@k8s-master promethues]# kubectl delete pod prometheus-5fb5c655dd-pcqzk 
pod "prometheus-5fb5c655dd-pcqzk" deleted

[root@k8s-master promethues]# kubectl get pod
NAME                          READY   STATUS    RESTARTS   AGE
node-exporter-kf4cs           1/1     Running   0          6m18s
node-exporter-q68q9           1/1     Running   0          6m18s
prometheus-5fb5c655dd-hrh7t   1/1     Running   0          3s

4、部署grafana展示数据

4.1 部署grafana

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana-core
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
      - image: grafana/grafana:5.0.0
        name: grafana-core
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 2
            memory: 2G
          requests:
            cpu: 1
            memory: 1G
        env:
          - name: GF_AUTH_BASIC_ENABLED
            value: "true"
          - name: GF_AUTH_ANONYMOUS_ENABLED
            value: "false"
        readinessProbe:
          httpGet:
            path: /login
            port: 3000
        volumeMounts:
        - name: grafana-persistent-storage
          mountPath: /var
      volumes:
      - name: grafana-persistent-storage
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  name: grafana
spec:
  type: NodePort
  ports:
    - port: 3000
  selector:
    app: grafana

[root@k8s-master promethues]# kubectl get pod
NAME                           READY   STATUS    RESTARTS   AGE
grafana-core-589f69bc8-hwh76   1/1     Running   0          58s
node-exporter-kf4cs            1/1     Running   0          16m
node-exporter-q68q9            1/1     Running   0          16m
prometheus-5fb5c655dd-hrh7t    1/1     Running   0          9m51s

[root@k8s-master promethues]# kubectl get svc
NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
grafana      NodePort    10.96.58.15    <none>        3000:30201/TCP   2m29s

4.2 对接promethues

默认用户名: admin, 密码admin
添加数据源

4.3 展示数据

• 主机展示模板 9276

四、通过mysql_exporter监控MySQL运行状态

1、部署MySQL数据库

apiVersion: apps/v1
kind: Deployment
metadata:
    name: test-mysql
spec:
    replicas: 1
    selector:
        matchLabels:
            app: mysql
    template:
        metadata:
            labels:
                app: mysql
        spec:
            restartPolicy: Always
            containers:
            - name: test-mysql
              image: mysql:5.7
              imagePullPolicy: IfNotPresent
              env:
              - name: MYSQL_ROOT_PASSWORD
                value: redhat
---
apiVersion: v1
kind: Service
metadata:
  name: test-mysql
spec:
  ports:
  - port: 3306
  selector:
    app: mysql

1.1 在MySQL创建监控用户

mysql> grant select,process,replication client on *.* to 'puser'@'%' identified by 'redhat';
Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)

2、部署mysql_exporter对接MySQL

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mysql-exporter
spec:
  replicas: 1
  selector:
    matchLabels:
      app: mysql-exporter
  template:
    metadata:
      labels:
        app: mysql-exporter
    spec:
      containers:
      - name: mysql-exporter
        image: registry.cn-beijing.aliyuncs.com/dotbalo/mysqld-exporter
        env:
        - name: DATA_SOURCE_NAME
          value: "puser:redhat@(test-mysql.default.svc.cluster.local:3306)/"
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9104
---
apiVersion: v1
kind: Service
metadata:
  name: mysql-exporter
spec:
  selector:
    app: mysql-exporter
  ports:
    - port: 9104

[root@k8s-master promethues]# kubectl get pod
NAME                             READY   STATUS    RESTARTS   AGE
mysql-exporter-55f465bf5-mg992   1/1     Running   0          4s

[root@k8s-master promethues]# kubectl get svc
NAME             TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
mysql-exporter   ClusterIP   10.96.142.34   <none>        9104/TCP         16s

3、编辑promethues配置文件，添加MySQL监控任务

      - job_name: 'test-mysql'
        static_configs:
        - targets: ["mysql-exporter.default.svc.cluster.local:9104"]

4、在grafana上展示MySQL数据