K8s可视化监控告警【4】--使用代理提供TLS+安全认证

1.证书密钥获取，两种方式选其一

#one
[root@k8s-master ~]#openssl genrsa -out cert.key 2048
[root@k8s-master ~]#openssl req -new -x509 -key cert.key -out cert.pem -days 3650 -subj /CN=www.example.com

#two
openssl genrsa > cert.key
openssl req -new -x509 -key cert.key > cert.pem

2.创建auth_basic

[root@k8s-master ~]#htpasswd -bc .htpasswd admin admin

3.创建secret

[root@k8s-master ~]#kubectl  create secret generic proxy-prometheus --from-file=cert.key --from-file=cert.pem --from-file=.htpasswd

4.app部署（configMap主要地址重写的IP为任意节点IP）

---
apiVersion: v1
kind: ReplicationController
metadata:
  name: proxy-prometheus
spec:
  replicas: 1
  template:
    metadata:
      name: proxy-prometheus
      labels:
        proxy: prometheus
    spec:
      tolerations:
      - key: node-role.kubernetes.io/master
        operation: Equal
        effect: NoSchedule  
      volumes:
      - name: certs
        secret:
          secretName: proxy-prometheus
      - name: proxy-conf
        configMap:
          name: proxy-prometheus-conf
      - name: china-time
        hostPath: 
          path: /usr/share/zoneinfo/Asia

      containers:
      - image: nginx
        name: proxy-prometheus
        ports:
        - name: test
          containerPort: 80
        - name: prom
          containerPort: 9090
        - name: alert
          containerPort: 9093
        - name: prom-tls
          containerPort: 38443
        - name: alert-tls
          containerPort: 39443
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: certs
          mountPath: /etc/nginx/certs/
          readOnly: true
        - name: proxy-conf
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf
        - name: proxy-conf
          mountPath: /etc/nginx/conf.d/default.conf
          subPath: default.conf
        - name: proxy-conf
          mountPath: /etc/nginx/conf.d/prometheus.conf
          subPath: prometheus.conf
        - name: proxy-conf
          mountPath: /etc/nginx/conf.d/alertmanager.conf
          subPath: alertmanager.conf
        - name: proxy-conf
          mountPath: /etc/nginx/conf.d/https.conf
          subPath: https.conf
        - name: china-time
          mountPath: /etc/localtime
          subPath: Shanghai

---
kind: Service
apiVersion: v1
metadata:
  labels:
    function1: nginx-proxy
    function2: tls
    function3: auth_basic
    app1: prometheus
    app2: alertmanager
  name: proxy-prometheus
  namespace: kube-system
spec:
  type: NodePort
  ports:
  - name: test
    port: 30080
    protocol: TCP
    targetPort: 80
    nodePort: 30080
  - name: tls-prom
    port: 38443
    protocol: TCP
    targetPort: 38443
    nodePort: 38443
  - name: tls-alert
    port: 39443
    protocol: TCP
    targetPort: 39443
    nodePort: 39443
  - name: prom
    port: 9090
    protocol: TCP
    targetPort: 9090
    nodePort: 9090
  - name: alert
    port: 9093
    protocol: TCP
    targetPort: 9093
    nodePort: 9093
  selector:
    proxy: prometheus


---
apiVersion: v1
kind: ConfigMap
metadata:
  name: proxy-prometheus-conf
  labels:
    proxy: prometheus
data:
  nginx.conf: |
    user  nginx;
    worker_processes  1;

    error_log  /var/log/nginx/error.log warn;
    pid        /var/run/nginx.pid;


    events {
        worker_connections  1024;
    }


    http {
        include       /etc/nginx/mime.types;
        default_type  application/octet-stream;
        server_tokens off;
        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                          '$status $body_bytes_sent "$http_referer" '
                          '"$http_user_agent" "$http_x_forwarded_for"';

        access_log  /var/log/nginx/access.log  main;

        sendfile        on;
        #tcp_nopush     on;

        keepalive_timeout  65;

        #gzip  on;

        include /etc/nginx/conf.d/*.conf;
    }
  prometheus.conf: |
    server {
        listen       38443 ssl;
        server_name  localhost;

        #charset koi8-r;
        #access_log  /var/log/nginx/host.access.log  main;
        ssl_certificate      certs/cert.pem;
        ssl_certificate_key  certs/cert.key;

        ssl_session_cache    shared:SSL:1m;
        ssl_session_timeout  5m;

        ssl_ciphers  HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers  on;

        location / {
            #root   /usr/share/nginx/html;
            #index  index.html index.htm;
            proxy_pass http://prometheus:9090/;
            auth_basic "Prometheus";
            auth_basic_user_file "certs/.htpasswd";
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        location ~ /\.ht {
           deny  all;
        }
    }
  alertmanager.conf: |
    server {
        listen       39443 ssl;
        server_name  localhost;

        #charset koi8-r;
        #access_log  /var/log/nginx/host.access.log  main;
        ssl_certificate      certs/cert.pem;
        ssl_certificate_key  certs/cert.key;

        ssl_session_cache    shared:SSL:1m;
        ssl_session_timeout  5m;

        ssl_ciphers  HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers  on;
        location / {
            #root   /usr/share/nginx/html;
            #index  index.html index.htm;
            proxy_pass http://alertmanager:9093/;
            auth_basic "Alertmanager";
            auth_basic_user_file "certs/.htpasswd";
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        location ~ /\.ht {
            deny  all;
        }
    }
  https.conf: |
    server {
        listen 9090;
        server_name localhost;

        #rewrite ^(.*)$ https://${server_name}:8443$1 permanent;
        rewrite ^(.*)$ https://节点IP:38443$1 permanent;
    }

    server {
        listen 9093;
        server_name localhost;

        #rewrite ^(.*)$ https://${server_name}:9443$1 permanent;
        rewrite ^(.*)$ https://节点IP:39443$1 permanent;
    }

  default.conf: |
    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;
        #access_log  /var/log/nginx/host.access.log  main;

        location / {
            root   /usr/share/nginx/html;
            index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        location ~ /\.ht {
           deny  all;
        }
    }

5.效果对比