keyval在K8S中的数据持久性验证

测试目的： K8S中keyval数据持久化。当pod故障或新建时能够不丢失keyval中数据

测试架构：

N+使用configmap获得配置文件

Keyval的state存放在节点目录

测试方式：

       验证nginx keyval功能正常

    删除deployment

       重新apply生成新pod

       测试新pod可以读取存放在节点目录的keyval信息

       Keyval信息无丢失

Dockerfile
FROM centos:7.9.2009

LABEL maintainer="N+  <ding@ding.com>"

RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \
    --mount=type=secret,id=nginx-key,dst=nginx-repo.key \
    # 1
    yum makecache \
    && mkdir -p /etc/ssl/nginx \
    && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \
    && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
    # Install the latest release of NGINX Plus and/or NGINX Plus modules
    && yum install ca-certificates -y \
    && yum install wget -y \
    && wget -P /etc/yum.repos.d https://cs.nginx.com/static/files/nginx-plus-7.4.repo \
    && yum install nginx-plus -y \
    && rm -rf /etc/ssl/nginx \
    && yum clean all \
    # 3
    && mv /etc/nginx/nginx.conf   /etc/nginx/nginx.conf.bak \
    && mv /etc/nginx/conf.d/default.conf  /etc/nginx/conf.d/default.conf.bak 
    #删除默认配置文件
    #传入修改的配置文件
COPY log-default.json /etc/nginx/

EXPOSE 80
STOPSIGNAL SIGQUIT
CMD ["nginx", "-g", "daemon off;"]

configmap-keyval.yaml
apiVersion: v1
kind: ConfigMap
metadata:
    name: nginx-config-keyval
    namespace: test1
data:
    nginx.conf: |
        user  nginx;
        worker_processes  auto;
        error_log  /var/log/nginx/error.log notice;
        pid        /var/run/nginx.pid;
        events {
            worker_connections  1024;
            }

        http {
            log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                              '$status "$request_body" $body_bytes_sent "$http_referer" '
                             '"$http_user_agent"  XFF:"$http_x_forwarded_for"';
           access_log  /var/log/nginx/access.log  main;
           sendfile        on;
           keepalive_timeout  65;
           fastcgi_connect_timeout 300;
           fastcgi_send_timeout 300;
           fastcgi_read_timeout 300;
           fastcgi_buffer_size 64k;
           fastcgi_buffers 4 64k;
           fastcgi_busy_buffers_size 128k;
           fastcgi_temp_file_write_size 128k;
           map $http_x_forwarded_for $client_ip {
               "~(?P<ip>\d+\.\d+\.\d+\.\d+)" $ip;
               "~(?P<ip>[^;]+)" $ip;
                default "255.255.255.255";
             }

           keyval_zone zone=one:2m state=/tmp/one.keyval;
           keyval $client_ip $target zone=one;
              
        server {
            listen 8888;
            location /api {
            api write=on;
             }

            location /example {
              if ($target != 1)
              { 
               return 403 "you can't access"; 
              }
               proxy_pass http://192.168.145.130:800/;
             }
         }
        }

deploy-keyval.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: npkeyval
  namespace: test1
  labels:
    app: npkeyval
spec:
  replicas: 1
  selector:
    matchLabels:
      app: npkeyval
  template:
    metadata:
      labels:
        app: npkeyval
    spec:
      nodeSelector:
        name: icmaster
      containers:
        - name: npkeyval
          image: nginxplus:keyval
          imagePullPolicy: Never
 #         command: [ "/bin/bash", "-ce", "tail -f /dev/null" ]
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
            - name: api
              containerPort: 8888
          volumeMounts:
            - mountPath: /etc/nginx
              readOnly: true
              name: nginx-config-keyval
            - mountPath: /tmp
              readOnly: false
              name: keyvalfile
      volumes:
        - name: nginx-config-keyval
          configMap:
            name: nginx-config-keyval
        - name: keyvalfile
          hostPath:
            path: /tmp/np

Nginx配置
sh-4.2# more /etc/nginx/nginx.conf 
user  nginx;
worker_processes  auto;
error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
    }

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status "$request_body" $body_bytes_sent "$http_referer" '
                     '"$http_user_agent"  XFF:"$http_x_forwarded_for"';
   access_log  /var/log/nginx/access.log  main;
   sendfile        on;
   keepalive_timeout  65;
   fastcgi_connect_timeout 300;
   fastcgi_send_timeout 300;
   fastcgi_read_timeout 300;
   fastcgi_buffer_size 64k;
   fastcgi_buffers 4 64k;
   fastcgi_busy_buffers_size 128k;
   fastcgi_temp_file_write_size 128k;
   map $http_x_forwarded_for $client_ip {
       "~(?P<ip>\d+\.\d+\.\d+\.\d+)" $ip;
       "~(?P<ip>[^;]+)" $ip;
        default "255.255.255.255";
     }

   keyval_zone zone=one:2m state=/tmp/one.keyval;
   keyval $client_ip $target zone=one;
      
server {
    listen 8888;
    location /api {
    api write=on;
     }

    location /example {
      if ($target != 1)
      { 
       return 403 "you can't access"; 
      }
       proxy_pass http://192.168.145.130:800/;
     }
 }
}

容器中keyval保存位置在/tmp/one.keyval

查看N+ pod，进入pod

查看pod中keyval信息

能阻挡黑名单XFF地址的访问

使用正常地址可以访问

删除POD

重新部署pod

黑名单地址无法访问

正常地址可以访问