Nginx1.9+LuaJIT+Kafka的点播监控系统实战

安装前的准备

Nginx1.9.9
LuaJIT-2.0.4
lua-nginx-module-0.10.2
ngx_devel_kit-0.3.0rc1
lua-resty-kafka
kafka_2.11-0.9.0.1

参考的这几篇文章方成此方案

Nginx与Lua

基于Lua+Kafka+Heka的Nginx Log实时监控系统

Lua如何读取nginx内置对象和方法

Kafka官方文档

nginx+lua+kafka实现日志统一收集汇总

提供一个相关软件和详细配置说明的网盘下载地址http://pan.baidu.com/s/1c2LmX36

1 安装LuaJIT

下载http://luajit.org/download.html
http://luajit.org/install.html
  命令如下：
tar zxf LuaJIT-2.0.4.tar.gz
cd LuaJIT-2.0.4
make PREFIX=/usr/local/LuaJIT
make install PREFIX=/usr/local/LuaJIT
echo "/usr/local/LuaJIT/lib" > /etc/ld.so.conf.d/usr_local_luajit_lib.conf
ldconfig
#注意环境变量!
export LUAJIT_LIB=/usr/local/LuaJIT/lib
export LUAJIT_INC=/usr/local/LuaJIT/include/luajit-2.0

2 安装lua-nginx-module

https://github.com/openresty/lua-nginx-module/tags
cd /usr/local
tar zxvf lua-nginx-module-0.10.2.tar.gz 

3 安装ngx_devel_kit

 

https://github.com/simpl/ngx_devel_kit/tags

 

http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
cd /usr/local
tar zxvf ngx_devel_kit-0.3.0rc1.tar.gz 

 4 安装编译Nginx

http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
给Nginx添加下面的参数，如果已经安装了Nginx则用
  nginx -V
   --add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1
如：
cd /usr/local/nginx-1.9.9
./configure  --prefix=/usr/local/nginxnew/ --with-http_stub_status_module --with-http_ssl_module --with-http_realip_module --add-module=/root/install/ngx_log_if-master --add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1

5 lua插件lua-resty-kafka

https://github.com/doujiang24/lua-resty-kafka
mkdir /usr/local/lua
上传lua-cjson-2.1.0.3.tar.gz到/usr/local/lua
上传lua-resty-kafka到/usr/local/lua

6 lua插件cjson

http://www.kyne.com.au/~mark/software/lua-cjson-manual.html
cd /usr/local/lua
tar zxvf lua-cjson-2.1.0.3.tar.gz 

7，nginx的配置

http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;
    log_format ht-video  '$request|$msec|$http_x_forwarded_for|$http_user_agent|$request_body';
    lua_package_path "/usr/local/lua/lua-resty-kafka/lib/?.lua;;";
        server{
                listen      8080;
                server_name 192.168.50.42;
                location / {
                        root /usr/local/nginx/htm;
                        index index.html;
                }
                location /ht{
                        #root   html;
                        #index  index.html index.htm;
                        if ( $request ~ "GET" ) {
                                #access_log logs/ht-video.log ht-video;
                                                access_log /htlog/ht-video.log ht-video;
                        }
 #                       proxy_set_header Host $http_host;
#                        proxy_pass http://localhost:6099;
                        client_max_body_size 8000M;
                        #error_page 405 =200 $1;
                }
        }

        server {
                listen       6099;
                server_name  localhost;
                location /ht {
                        root   /usr/local/nginx;
                        index  index.html index.htm;
                         client_max_body_size 8000M;
                        error_page 405 =200 $1;
                }
        }
    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;
        #为了方便调试，关闭了lua_code_cache，如果是生产环境，应该开启它。
        lua_code_cache off;
        location / {
            root   html;
            index  index.html index.htm;
        }

       location /lua {
                default_type 'text/plain';
                content_by_lua 'ngx.say("hello, lua")';
       }
       location = /lua-v {
            content_by_lua '
                ngx.header.content_type = "text/plain";
                if jit then
                        ngx.say(jit.version)
                else
                        ngx.say(_VERSION)
                end
             ';
        }
       location /ht3/video/p {
            content_by_lua '
                ngx.header.content_type = "text/plain";
                --local cjson = require "cjson"
                --local client = require "resty.kafka.client"
                local producer = require "resty.kafka.producer"

 

                local broker_list = {
                    { host = "192.168.50.42", port = 9092 }
                }
                local broker_list_host_ip = {
                    { host = "kafka1.livecourse.com", ip = "192.168.50.42" },
                }

 

                local key = "key"
                --local message = $request|$msec|$remote_addrx|$http_user_agent|$request_body

 

                local myIP = ngx.req.get_headers()["X-Real-IP"]
                if myIP == nil then
                   myIP = ngx.req.get_headers()["x_forwarded_for"]
                end
                if myIP == nil then
                    myIP = ngx.var.remote_addr
                end
                local h = ngx.req.get_headers()

 

                local message = ngx.req.get_method() .. " " .. ngx.var.uri
                if ngx.var.args  ~= nil then
                  message = message .. "?" .. ngx.var.args .. "|"
                end
                message = message .. ngx.now() .. "|"
                message = message .. myIP .. "|"
                message = message .. h["user-agent"] .. "|"
                local bodyData = ngx.req.get_body_data()
                if bodyData  == nil then
                bodyData = "-"
                end
                message = message .. bodyData

 

                -- this is async producer_type and bp will be reused in the whole nginx worker
                local bp = producer:new(broker_list, { producer_type = "async" , broker_list_host_ip = broker_list_host_ip,refresh_interval = 3000})
                local ok, err = bp:send("test", key, message)

              -- 定义kafka异步生产者
                -- 发送日志消息,send第二个参数key,用于kafka路由控制:
                -- key为nill(空)时，一段时间向同一partition写入数据
                -- 指定key，按照key的hash写入到对应的partition

               -- 指定key，按照key的hash写入到对应的partition

                if not ok then
                    --ngx.say("send err:", err)
                    ngx.say("void(1);");
                    return
                end
                ngx.say("void(0);");
                --ngx.say("send success, ok:", ok)
            ';
      
        }
        #error_page  404              /404.html;

 

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
           root   html;
        } 
    }

}

 注意配置文件标红的的，第一个标红的为kafka的安装服务器地址和端口，如果有多台则配置多个，第二个标红的test为kafka的topic名称

8 完成

请求Url：http://localhost/ht3/video/p?a=bbbb 就能将日志内容发送到kafka