NGINX自动清理180天之前的日志

需求描述

日志每天会以天为单位产生一个日志，不清理的话会越来越多。这里写一个Lua自定定时清理日志目录下的日志文件。

 依赖安装

安装 lfs 模块

yum install luarocks
yum install lua-devel

luarocks install luafilesystem

创建模拟旧文件

创建了一个1月的旧文件

[root@iZbp1xxxxxxxxxq7ioZ logs]# ll
total 24
-rw-r--r-- 1 app  app  5124 Jan 16 15:30 access_api_.json_2024-01-16
-rw-r--r-- 1 app  app  8540 Aug 16  2024 access_api_.json_2024-08-16
-rw-r--r-- 1 app  app     0 Aug 14  2024 api-error.log
-rw-r--r-- 1 root root  747 Jan 16 16:41 luatest.lua

配置文件及脚本

检查配置文件后开始重启，中间的Lua脚本会24小时执行一次，检查目录中是否有180天的日志文件。

user  root;
worker_processes  6;
events {
    worker_connections  30000;
}
http {
# lua环境变量
lua_package_cpath "/usr/lib64/lua/5.1/?.so;;";
    ###
    init_worker_by_lua_block {
        local lfs = require "lfs"

        local function clean_old_logs(premature)
            if premature then
                return
            end

            -- 日志目录路径
            local log_path = "/data/logs/"
            -- 当前时间
            local current_time = os.time()

            -- 定义保留天数
            local retain_days = 180
            local retain_seconds = retain_days * 24 * 60 * 60

            -- 遍历日志目录
            for file in lfs.dir(log_path) do
                -- 检查文件名是否符合日志格式
                if file:match("access_api.json_%d%d%d%d%-%d%d%-%d%d") then
                    -- 获取文件的完整路径
                    local file_path = log_path .. file
                    -- 获取文件的属性
                    local attr = lfs.attributes(file_path)

                    if attr then
                        local file_age = current_time - attr.modification
                        ngx.log(ngx.NOTICE, "File: ", file_path, " Age: ", file_age, " Retain seconds: ", retain_seconds)

                        -- 如果文件超过30天，则删除
                        if file_age > retain_seconds then
                            local success, err = os.remove(file_path)
                            if success then
                                ngx.log(ngx.NOTICE, "Deleted old log file: ", file_path)
                            else
                                ngx.log(ngx.ERR, "Failed to delete log file: ", file_path, " Error: ", err)
                            end
                        end
                    else
                        ngx.log(ngx.ERR, "Failed to get attributes for file: ", file_path)
                    end
                end
            end
        end

        -- 设置定时器，每86400秒（即24小时）执行一次
        local ok, err = ngx.timer.every(86400, clean_old_logs)
        if not ok then
            ngx.log(ngx.ERR, "Failed to create timer: ", err)
        end
    }
    ###
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;
    ###日志格式
    log_format json  escape=json '{"timestamp":"$time_iso8601",'
                        '"remote_addr": "$remote_addr", '
                        '"referer": "$http_referer", '
                        '"request": "$request", '
                        '"status": $status, '
                        '"bytes": $body_bytes_sent, '
                        '"agent": "$http_user_agent", '
                        '"x_forwarded": "$http_x_forwarded_for", '
                        '"up_addr": "$upstream_addr",'
                        '"up_host": "$upstream_http_host",'
                        '"up_resp_time": "$upstream_response_time",'
                        '"request_time": "$request_time",'
                        '"request_GlobalId": "$http_GlobalId",'
                        '"response_GlobalId": "$sent_http_GlobalId"'
                        '"response_body": "$resp_body",'
                        '"request_body": "$request_body"'
                        ' }';
    ###日志按天分割
    map $time_iso8601 $logdate{
            '~^(?<ymd>\d{4}-\d{2}-\d{2})' $ymd;
            default 'date-not-found';
    }
    ###
    charset  utf-8 ;
    gzip  on;
    # 后端IP地址
    upstream api-prod {
        server 10.66.66.86:8501 max_fails=5 fail_timeout=30s;
        server 10.66.66.88:8501 max_fails=5 fail_timeout=30s;

    }

    server {
        listen       80 ;
        listen       7309 ;
        server_name  api.xxxxxx.cn api-test.xxxxxx.cn;
        charset  utf-8 ;
        #日志配置
        lua_need_request_body on;
        set $resp_body "";
        body_filter_by_lua '
            local resp_body = string.sub(ngx.arg[1], 1, 1000)
            ngx.ctx.buffered = (ngx.ctx.buffered or "") .. resp_body
            if ngx.arg[2] then
               ngx.var.resp_body = ngx.ctx.buffered
            end
        ';

        location / {
           proxy_pass http://api-prod;

        # 屏蔽 SLBHealthCheck 和 Blackbox Exporter/0.21.1 的 404 请求日志
        if ($http_user_agent ~* "(SLBHealthCheck|Blackbox Exporter/0.21.1)") {
            access_log off;
            return 200;
        }
        }
   access_log /data/logs/access_api.json_$logdate json;
   error_log /data/logs/api-error.log error;
}
}

验证

180天之前的日志文件没有了。

[root@iZbpxxxxxxxxxxxoZ logs]# ll /data/logs
total 16
-rw-r--r-- 1 root root 8540 Aug 16 16:37 access_api.json_2024-08-16
-rw-r--r-- 1 root root    0 Aug 14 16:36 api-error.log
-rw-r--r-- 1 root root  747 Jan 16  2024 luatest.lua