prometheus监控k8s_etcd

一,普罗米修斯监控携带metric接口的服务的流程：

 普罗米修斯监控携带metric接口的服务的流程：

 1、通过EndPrints获取需要监控的ETCD的地址

 2、创建Service，给予集群内部的ServiceMoniter使用

 3、创建ServiceMoniter部署需要访问证书，给予prometheus-k8s-0来使用

 4、重启普罗米修斯监控Pod（prometheus-k8s-0）,载入监控项
二,测试metric接口可用
# 出现内容 代表metric接口可用
curl -k --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key https://127.0.0.1:2379/metrics

三, 流程
1、通过EndPrints获取需要监控的ETCD的地址
[root@\ k8s-m-01~]# mkdir etcd
[root@\ k8s-m-01~]# cd etcd/

[root@\ k8s-m-01~/etcd]# vim etcd-endpoints.yaml
kind: Endpoints
apiVersion: v1
metadata:
  namespace: kube-system
  name: etcd-moniter
  labels:
    k8s: etcd
subsets:
  - addresses:
      - ip: "192.168.15.31"
    ports:
      - port: 2379
        protocol: TCP
        name: etcd

# 部署
[root@\ k8s-m-01~/etcd]# kubectl apply -f etcd-endpoints.yaml 

# 查看部署结果
[root@\ k8s-m-01~/etcd]# kubectl get endpoints -n kube-system 
NAME           ENDPOINTS                                                                 AGE
etcd-moniter   192.168.15.31:2379                                                        31s

2、创建Service，给予集群内部的ServiceMoniter使用
[root@\ k8s-m-01~/etcd]# vim etcd-service.yaml
kind: Service
apiVersion: v1
metadata:
  namespace: kube-system
  name: etcd-moniter
  labels:
    k8s: etcd
spec:
  ports:
    - port: 2379
      targetPort: 2379
      name: etcd
      protocol: TCP

[root@\ k8s-m-01~/etcd]# kubectl apply -f etcd-service.yaml

[root@\ k8s-m-01~/etcd]# kubectl get svc -n kube-system 
NAME           TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                        AGE
etcd-moniter   ClusterIP   10.111.32.236   <none>        2379/TCP                       15s



# 拿service的ip测试 是否能获取到内容（获取即代表成功）
[root@\ k8s-m-01~/etcd]# curl -k --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key https://10.111.32.236:2379/metrics


3、创建ServiceMoniter部署需要访问证书
[root@\ k8s-m-01~/etcd]# vim etcd-servicemonitor.yaml
kind: ServiceMonitor
apiVersion: monitoring.coreos.com/v1
metadata:
  labels:
    k8s: etcd
  name: etcd-monitor
  namespace: monitoring
spec:
  endpoints:
  - interval: 3s
    port: etcd
    scheme: https
    tlsConfig:
      caFile: /etc/prometheus/secrets/etcd-certs/ca.crt
      certFile: /etc/prometheus/secrets/etcd-certs/peer.crt
      keyFile: /etc/prometheus/secrets/etcd-certs/peer.key
      insecureSkipVerify: true
  selector:
    matchLabels:
      k8s: etcd
  namespaceSelector:
    matchNames:
      - "kube-system"


[root@\ k8s-m-01~/etcd]# kubectl apply -f etcd-servicemonitor.yaml

[root@\ k8s-m-01~/etcd]# kubectl get ServiceMonitor -n monitoring 
NAME                      AGE
etcd-monitor              25s

4、重启普罗米修斯监控Pod（prometheus-k8s-0）,载入监控项
# 创建一个secrets，用来保存prometheus监控的etcd的证书
[root@\ k8s-m-01~/etcd]# kubectl create secret generic etcd-certs -n monitoring --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/peer.crt --from-file=/etc/kubernetes/pki/etcd/peer.key
secret/etcd-certs created



# 创建监控
[root@\ k8s-m-01~/etcd]# vim prometheus.yaml
kind: Prometheus
apiVersion: monitoring.coreos.com/v1
metadata:
  labels:
    prometheus: k8s
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
      - name: alertmanager-main
        namespace: monitoring
        port: web
      - name: alertmanager-main-etcd
        namespace: kube-system
        port: etcd
  image: quay.io/prometheus/prometheus:v2.15.2
  nodeSelector:
    kubernetes.io/os: linux
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleSelector:
    matchLabels:
      prometheus: k8s
      role: alert-rules
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  version: v2.15.2
  secrets:
    - etcd-certs


[root@\ k8s-m-01~/etcd]# kubectl apply -f prometheus.yaml 


[root@\ k8s-m-01~/etcd]# kubectl get pods -n monitoring 
NAME                                   READY   STATUS    RESTARTS   AGE
prometheus-k8s-0                       3/3     Running   1          56s
prometheus-k8s-1                       3/3     Running   2          114s