Lua+Nginx+Kafka

最新推荐文章于 2024-02-26 14:23:03 发布

zhuhailong

最新推荐文章于 2024-02-26 14:23:03 发布

阅读量3.3k

点赞数

分类专栏： Lua 文章标签： Lua nginx 配置文件详解 nginx.conf kafka

本文链接：https://blog.csdn.net/u011317245/article/details/52688785

版权

Lua 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

http {
    include       mime.types;
    default_type 'text/plain';
    log_format  main  '$http_x_forwarded_for $remote_addr - $remote_user [$time_local] "$request" '
              '$status $body_bytes_sent "$http_referer" '
              '"$http_user_agent" $http_x_forwarded_for';
    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';
    log_format icjson '{"@timestamp":"$time_iso8601",'
               '"@version":"1",'
               '"host":"$server_addr",'
               '"http_x_forwarded_for":"$http_x_forwarded_for",'
               '"remote_addr":"$remote_addr",'
               '"size":"$body_bytes_sent",'
               '"request_body":$request_body,'
               '"responsetime":"$request_time",'
               '"domain":"$host",'
               '"url.raw":"$uri",'
               '"http_accept":"$http_accept",'
               '"gomeplus_token":"$http_x_gomeplus_token",'
               '"gomeplus_access_token":"$http_x_gomeplus_access_token",'
               '"user_agent":"$http_user_agent",'
               '"method":"$request_method $scheme://$http_host$request_uri $server_protocol" ,'
               '"status":"$status"}';

    lua_shared_dict  localstorage 64m;
    resolver xx.xxx.xx.xxx xx.xxx.xx.xxx xx.xxx.xxx.xxx xx.xxx.xxx.xxx ; 
    #-------lua_package_path '/gomeo2o/www/app_log/?.lua;;';
    lua_package_path '/gomeo2o/www/app_log/?.lua;/gomeo2o/software/lua-resty-kafka-master/lib/?.lua;';
    init_by_lua_file '/gomeo2o/www/app_log/lib/init.lua';




    server {
       listen       80;

       location ~ /log? {
           content_by_lua_file '/gomeo2o/www/app_log/js.lua';
    	   access_log  logs/js.access.log  main;
           log_by_lua_file '/gomeo2o/www/app_log/kafkaJs.lua';
       }

       location = /postlog{ 
            lua_need_request_body on ;
    	    content_by_lua_file '/gomeo2o/www/app_log/log.lua';
            access_log  logs/json.access.log  icjson ;
            log_by_lua_file '/gomeo2o/www/app_log/kafkaJson.lua';
       }

    }
}

init.lua

client = require "resty.kafka.client"
producer = require "resty.kafka.producer"

Lua中不加前缀定义为全局变量，定义本地变量使用local前缀 例如：local num = 1
本地变量作用域：
1)本地变量定义在一个函数体中，那么作用域就在函数体中。
2)本地变量定义在一个控制结构中，那么作用域就在这个控制结构中
3)本地变量定义在一个文件中，那么作用域就在这个文件中
4)本地变量定义在命令行中，那么一条完整的命令就是一个chunk
> local i=1
> print(i)
nil

KafkaJs.lua

    local broker_list = {
          { host = "bj01.test.com", port = 9092 },
          { host = "bj02.test.com", port = 9092 },
          { host = "bj03.test.com", port = 9092 }
    }

    local http_x_forwarded_for = ngx.var.http_x_forwarded_for 
    if http_x_forwarded_for == nil then
        http_x_forwarded_for = "-"
    end

    local remote_addr = ngx.var.remote_addr
    if remote_addr == nil then
        remote_addr = "-"
    end

    local remote_user = ngx.var.remote_user
    if remote_user == nil then
        remote_user = "-"
    end

    local time_local = ngx.var.time_local
    if time_local == nil then
        time_local = "-"
    end 

    local request = ngx.var.request
    if request == nil then
        request = "-"
    end
    
    local status = ngx.var.status
    if status == nil then
        status = "-"
    end
    
    local body_bytes_sent = ngx.var.body_bytes_sent
    if body_bytes_sent == nil then
        body_bytes_sent = "-"
    end

    local http_referer = ngx.var.http_referer
    if http_referer == nil then
        http_referer = "-"
    end

    local http_user_agent = ngx.var.http_user_agent
    if http_user_agent == nil then
        http_user_agent = "-"
    end


    local message = http_x_forwarded_for .. " " .. remote_addr .. " " .. "-" .. " " .. remote_user .. " " .. "[" .. time_local .. "]" .. " " .. "\"" .. request .. "\"" .. " " .. status .. " " .. body_bytes_sent .. " " .. "\"" .. http_referer .. "\""  .. " " .. "\"" .. http_user_agent .. "\"" .. " " .. http_x_forwarded_for ;
 

    local file,error = io.open("/gomeo2o/www/app_log/lua_js_access.log","a+")
    file:write(message.."\n");
    file:flush();
    file:close();
    


    local bp = producer:new(broker_list, { producer_type = "async" })
    local ok, err = bp:send("meixin_js_log_lua",null, message)
    if not ok then
           ngx.log(ngx.ERR, err)
           return
    end
    --ngx.say("send success, ok:", ok)

KafkaJson.lua

   local broker_list = {
          { host = "bj01.test.com", port = 9092 },
          { host = "bj02.test.com", port = 9092 },
          { host = "bj03.test.com", port = 9092 }
    }

    
    local time_iso8601 = ngx.var.time_iso8601
    if time_iso8601 == nil then
        time_iso8601 = "-"
    end

    local server_addr = ngx.var.server_addr
    if server_addr == nil then
        server_addr = "-"
    end

    local http_x_forwarded_for = ngx.var.http_x_forwarded_for
    if http_x_forwarded_for == nil then
        http_x_forwarded_for = "-"
    end

    local remote_addr = ngx.var.remote_addr
    if remote_addr == nil then
        remote_addr = "-"
    end
  
    local body_bytes_sent = ngx.var.body_bytes_sent
    if body_bytes_sent == nil then
        body_bytes_sent = "-"
    end

    local request_body = ngx.var.request_body
    if request_body == nil then
       request_body = "{}"
    end

    request_body = string.gsub(tostring(request_body), "\n", "")

    local request_time = ngx.var.request_time
    if request_time == nil then
        request_time = "-"
    end

    local host = ngx.var.host
    if host == nil then
        host = "-" 
    end

    local uri = ngx.var.uri
    if uri == nil then
        uri = "-"
    end

    local http_accept = ngx.var.http_accept
    if http_accept == nil then
        http_accept = "-"
    end

    local http_x_gomeplus_token = ngx.var.http_x_gomeplus_token
    if http_x_gomeplus_token == nil then
        http_x_gomeplus_token = "-"
    end

    local http_x_gomeplus_access_token = ngx.var.http_x_gomeplus_access_token
    if http_x_gomeplus_access_token == nil then
        http_x_gomeplus_access_token = "-"
    end

    local http_user_agent = ngx.var.http_user_agent
    if http_user_agent == nil then
        http_user_agent = "-"
    end

    local request_method = ngx.var.request_method
    if request_method == nil then
        request_method = "-"
    end

    local scheme = ngx.var.scheme
    if scheme == nil then
        scheme = "-"
    end 

    local http_host = ngx.var.http_host
    if http_host == nil then
        http_host = "-"
    end

    local request_uri = ngx.var.request_uri
    if request_uri == nil then
        request_uri = "-"
    end

    local server_protocol = ngx.var.server_protocol
    if server_protocol == nil then
        server_protocol = "-"
    end

    local status = ngx.var.status
    if status == nil then
        status = "-"
    end


    local message = '{"@timestamp":"' .. time_iso8601 .. '",' .. 
               '"@version":"1",' .. 
               '"host":"' .. server_addr .. '",' .. 
               '"http_x_forwarded_for":"' .. http_x_forwarded_for .. '",' .. 
               '"remote_addr":"' .. remote_addr ..'",' .. 
               '"size":"' .. body_bytes_sent .. '",' .. 
               '"request_body":' .. request_body .. ',' .. 
               '"responsetime":"' .. request_time .. '",' .. 
               '"domain":"' .. host .. '",' .. 
               '"url.raw":"' .. uri .. '",' .. 
               '"http_accept":"' .. http_accept .. '",' .. 
               '"gomeplus_token":"' .. http_x_gomeplus_token ..'",' .. 
               '"gomeplus_access_token":"' .. http_x_gomeplus_access_token .. '",' .. 
               '"user_agent":"' .. http_user_agent .. '",' .. 
               '"method":"' .. request_method .. ' ' .. scheme .. '://' .. http_host .. '' .. request_uri .. ' ' .. server_protocol .. '" ,' .. 
               '"status":"' .. status ..'"}'; 

    local file,error = io.open("/gomeo2o/www/app_log/lua_json_access.log","a+")
    file:write(message.."\n");
    file:flush();
    file:close();


    local bp = producer:new(broker_list, { producer_type = "async" })
    local ok, err = bp:send("meixin_app_log_lua",null, message)
    if not ok then
           ngx.say("send err:", err)
           return
    end