prom-operator thanos部署及多集群纳管

thanos原理架构详解参考打造云原生大型分布式监控系统(二): Thanos 架构详解
这里使用promtheus-operator做部署，minio做对象存储，promethues-operator部署不在这里描述。

一、minio部署

1、statefulsets部署

# 创建minio用户密码，使用base64加密
[root@node 二 7月 11 17:35 thanos]# cat minio-secret.yaml 
apiVersion: v1                  # kubectl explain secret 查看
kind: Secret
metadata:
  name: minio                # Secret 名称
  namespace: monitoring
type: Opaque                    # Secret 的类型
data:
  password: bWluaW8xMjMK            # 密码
  username: bWluaW8K    # 用户名

# 部署statefulsets minio,创建pvc
[root@node 二 7月 11 17:37 thanos]# cat minio-dev.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
#  labels:
#    app: minio
  name: minio
  namespace: monitoring
spec:
  serviceName: minio
  replicas: 2
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      containers:
      - name: minio
        env:
        - name: MINIO_ROOT_USER
          valueFrom:
            secretKeyRef:
              name: minio
              key: username
        - name: MINIO_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: minio
              key: password
 #         value: "******"
        image: quay.io/minio/minio:latest
        imagePullPolicy: IfNotPresent
        command:
          - /bin/sh
          - -c
          - minio server --console-address ":5000" http://minio-{0..1}.minio.monitoring.svc.cluster.local/data/minio
        ports:
        - name: data
          containerPort: 9000
          protocol: "TCP"
        - name: console
          containerPort: 5000
          protocol: "TCP"
        volumeMounts:
        - name: data
          mountPath: /data
        - name: date-config
          mountPath: /etc/localtime
      volumes:
        - name: date-config
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
        - ReadWriteMany
      resources:
        requests:
          storage: 1Gi
      storageClassName: nfs-storage

# 创建service
[root@node 二 7月 11 17:35 thanos]# cat minio-svc.yaml 
apiVersion: v1
kind: Service
metadata:
  name: minio
  labels:
    app: minio
spec:
  clusterIP: None
  ports:
    - port: 9000
      name: data
    - port: 5000
      name: console
  selector:
    app: minio
---
apiVersion: v1
kind: Service
metadata:
  name: minio-test
spec:
  type: NodePort
  ports:
  - port: 9000
    name: data
    nodePort: 30109
    targetPort: 9000
    protocol: TCP
  - port: 5000
    name: console
    nodePort: 30110
    targetPort: 5000
    protocol: TCP
  selector:
    app: minio

2、minio创建bucket

# 登录minio，创建bucket，将access_key、 bucket、secret_key输出成thanos-objectstorage secret文件

[root@node 二 7月 11 17:41 thanos]# cat secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: thanos-objectstorage
  namespace: monitoring
type: Opaque
stringData:
  thanos.yaml: |-
    type: s3
    config:
      bucket: thanos
      endpoint: 192.168.200.101:30109
      insecure: true
      access_key: tV7t4lZ7AVeqywfqXi7x
      secret_key: fE0r3mjVYIqon9fV8y1lOAuYRS3LYItH4dMMUmbG

二、部署thanos组件

1、部署thanos组件

# 创建thanos-objectstorage secret

# prometheus资源添加thanos(thanos-sidecar)配置
# kubectl edit prometheus k8s -n monitoring
# 最下方添加
  thanos: # 添加 thanos 配置
    image: thanosio/thanos:v0.26.0
    resources:
      limits:
        cpu: 500m
        memory: 500Mi
      requests:
        cpu: 100m
        memory: 500Mi
    objectStorageConfig:
      key: thanos.yaml
      name: thanos-objectstorage

# 创建store                            
[root@node 二 7月 11 17:46 thanos]# cat store.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-store
  namespace: monitoring
  labels:
    app: thanos-store
spec:
  replicas: 1
  selector:
    matchLabels:
      app: thanos-store
  serviceName: thanos-store
  template:
    metadata:
      labels:
        app: thanos-store
        thanos-store-api: "true"
    spec:
      containers:
        - name: thanos
          image: thanosio/thanos:v0.26.0
          args:
          - "store"
          - "--log.level=debug"
          - "--data-dir=/data"
          - "--objstore.config-file=/etc/secret/thanos.yaml"
          - "--index-cache-size=500MB"
          - "--chunk-pool-size=500MB"
          ports:
          - name: http
            containerPort: 10902
          - name: grpc
            containerPort: 10901
          livenessProbe:
            httpGet:
              port: 10902
              path: /-/healthy
            initialDelaySeconds: 10
          readinessProbe:
            httpGet:
              port: 10902
              path: /-/ready
            initialDelaySeconds: 15
          volumeMounts:
            - name: object-storage-config
              mountPath: /etc/secret
              readOnly: false
      volumes:
        - name: object-storage-config
          secret:
            secretName: thanos-objectstorage
---
apiVersion: v1
kind: Service
metadata:
  name: thanos-store
  namespace: monitoring
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: grpc
      port: 10901
      targetPort: grpc
  selector:
    app: thanos-store

# 创建compactor
[root@node 二 7月 11 17:47 thanos]# cat compactor.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-compactor
  namespace: monitoring
  labels:
    app: thanos-compactor
spec:
  replicas: 1
  selector:
    matchLabels:
      app: thanos-compactor
  serviceName: thanos-compactor
  template:
    metadata:
      labels:
        app: thanos-compactor
    spec:
      containers:
      - name: thanos
        image: thanosio/thanos:v0.26.0
        args:
        - "compact"
        - "--log.level=debug"
        - "--data-dir=/data"
        - "--objstore.config-file=/etc/secret/thanos.yaml"
        - "--wait"
        ports:
        - name: http
          containerPort: 10902
        livenessProbe:
          httpGet:
            port: 10902
            path: /-/healthy
          initialDelaySeconds: 10
        readinessProbe:
          httpGet:
            port: 10902
            path: /-/ready
          initialDelaySeconds: 15
        volumeMounts:
        - name: object-storage-config
          mountPath: /etc/secret
          readOnly: false
      volumes:
      - name: object-storage-config
        secret:
          secretName: thanos-objectstorage

# 创建querier
[root@node 二 7月 11 17:47 thanos]# cat querier.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  selector:
    matchLabels:
      app: thanos-querier
  template:
    metadata:
      labels:
        app: thanos-querier
    spec:
      containers:
      - name: thanos
        image: thanosio/thanos:v0.26.0
        args:
        - query
        - --log.level=debug
        - --query.replica-label=prometheus_replica # 注意这行
        - --store=dnssrv+prometheus-operated:10901 # 注意这行
        - --store=dnssrv+thanos-store:10901 # 注意这行，先注释，一会儿再取消注释
        ports:
        - name: http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        resources:
          requests:
            memory: "1Gi"
            cpu: "500m"
          limits:
            memory: "1Gi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: http
          initialDelaySeconds: 10
        readinessProbe:
          httpGet:
            path: /-/healthy
            port: http
          initialDelaySeconds: 15
---
apiVersion: v1
kind: Service
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  ports:
  - port: 9090
    nodePort: 30104
    protocol: TCP
    targetPort: http
    name: http
  selector:
    app: thanos-querier
  type: NodePort

三、多集群prometheus thanos sidecar采集

# 根据二中在不同集群添加prometheus资源添加thanos(thanos-sidecar)配置
# kubectl edit prometheus k8s -n monitoring
# 最下方添加
  thanos: # 添加 thanos 配置
    image: thanosio/thanos:v0.26.0
    resources:
      limits:
        cpu: 500m
        memory: 500Mi
      requests:
        cpu: 100m
        memory: 500Mi
    objectStorageConfig:
      key: thanos.yaml
      name: thanos-objectstorage


# prometheus-operated service创建10901端口
[root@node 二 7月 11 17:51 thanos]# kubectl get svc -n monitoring  prometheus-operated -o yaml
apiVersion: v1
kind: Service
metadata:

  labels:
    operated-prometheus: "true"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1

    manager: operator
    operation: Update

  name: prometheus-operated
  namespace: monitoring
  ownerReferences:
  - apiVersion: monitoring.coreos.com/v1
    kind: Prometheus
    name: mysql

  - apiVersion: monitoring.coreos.com/v1
    kind: Prometheus
    name: k8s


spec:
  clusterIP: None
  ports:
  - name: web
    port: 9090
    protocol: TCP
    targetPort: web
  - name: grpc
    port: 10901
    protocol: TCP
    targetPort: grpc
  selector:
    app.kubernetes.io/name: prometheus
  sessionAffinity: None
  type: ClusterIP
  

# 创建prometheus NodePort service
[root@node 二 7月 11 17:55 thanos]# cat thanos-sidecar-svc.yaml 
apiVersion: v1
kind: Service
metadata:
  labels:
      app: prometheus
  name: thanos-sidecar
  namespace: monitoring
spec:
  type: NodePort

  ports:
    - name: grpc
      port: 10901
      targetPort: grpc
      nodePort: 30901
  selector:
    app: prometheus

# 在thanos-querier所在集群的配置里添加 - --store=192.168.200.101:30901(新集群的ip+端口)
[root@node 二 7月 11 17:47 thanos]# cat querier.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  selector:
    matchLabels:
      app: thanos-querier
  template:
    metadata:
      labels:
        app: thanos-querier
    spec:
      containers:
      - name: thanos
        image: thanosio/thanos:v0.26.0
        args:
        - query
        - --log.level=debug
        - --query.replica-label=prometheus_replica # 注意这行
        - --store=dnssrv+prometheus-operated:10901 # 注意这行
        - --store=dnssrv+thanos-store:10901 # 注意这行，先注释，一会儿再取消注释
        - --store=192.168.200.101:30901
        ports:
        - name: http
          containerPort: 10902
        - name: grpc
          containerPort: 10901
        resources:
          requests:
            memory: "1Gi"
            cpu: "500m"
          limits:
            memory: "1Gi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: http
          initialDelaySeconds: 10
        readinessProbe:
          httpGet:
            path: /-/healthy
            port: http
          initialDelaySeconds: 15
---
apiVersion: v1
kind: Service
metadata:
  name: thanos-querier
  namespace: monitoring
  labels:
    app: thanos-querier
spec:
  ports:
  - port: 9090
    nodePort: 30104
    protocol: TCP
    targetPort: http
    name: http
  selector:
    app: thanos-querier
  type: NodePort