基于ngx_kafka_module插件做日志收集（git安装）

安装git

[root@10 nginx]# yum install -y git
...
作为依赖被安装:
  perl-Error.noarch 1:0.17020-2.el7                                                       perl-Git.noarch 0:1.8.3.1-23.el7_8                                                       perl-TermReadKey.x86_64 0:2.30-20.el7     
完毕！

安装kafka的c客户端

• 克隆代码到本地

[root@10 software]# pwd
/home/laosan/software
[root@10 software]# git clone git://github.com/edenhill/librdkafka
正克隆到 'librdkafka'...
remote: Enumerating objects: 24, done.
remote: Counting objects: 100% (24/24), done.
remote: Compressing objects: 100% (18/18), done.
remote: Total 22737 (delta 8), reused 9 (delta 6), pack-reused 22713
接收对象中: 100% (22737/22737), 15.26 MiB | 12.00 KiB/s, done.
处理 delta 中: 100% (17421/17421), done.
[root@10 software]# 

• 安装依赖

[root@10 librdkafka]# yum install -y gcc gcc-c++ pcre-devel zlib-devel
...已安装:
  gcc-c++.x86_64 0:4.8.5-39.el7                                                           
作为依赖被安装:
  libstdc++-devel.x86_64 0:4.8.5-39.el7                                                   
作为依赖被升级:
  libstdc++.x86_64 0:4.8.5-39.el7                                                         
完毕！

• 进入librdkafka编译

[root@10 librdkafka]# pwd
/home/laosan/software/librdkafka
[root@10 librdkafka]# ./configure
...
Generated config.cache
Now type 'make' to buil
[root@10 librdkafka]# make && make install
make[1]: 进入目录“/home/laosan/software/librdkafka/src”
...

安装nginx整合kafka的插件

• 克隆代码到本地

[root@10 software]# pwd
/home/laosan/software
[root@10 software]# git clone git://github.com/brg-liuwei/ngx_kafka_module
正克隆到 'ngx_kafka_module'...
remote: Enumerating objects: 154, done.
remote: Total 154 (delta 0), reused 0 (delta 0), pack-reused 154
接收对象中: 100% (154/154), 34.16 KiB | 17.00 KiB/s, done.
处理 delta 中: 100% (79/79), done.

• 编译nginx及插件

[root@10 nginx-1.17.8]# pwd
/home/laosan/software/nginx-1.17.8
[root@10 nginx-1.17.8]# ./configure --add-module=/home/laosan/software/ngx_kafka_module/
...
[root@10 nginx-1.17.8]# make && make install
...

• 加载so库

[root@10 nginx-1.17.8]# echo "/usr/local/lib" >> /etc/ld.so.conf
[root@10 nginx-1.17.8]# ldconfig
[root@10 nginx-1.17.8]# /usr/local/nginx/sbin/nginx -V
nginx version: nginx/1.17.8
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-39) (GCC) 
configure arguments: --add-module=/home/laosan/software/ngx_kafka_module/
[root@10 nginx-1.17.8]# 

nginx配置文件

[root@10 conf]# pwd
/home/laosan/software/nginx-1.17.8/conf
[root@10 conf]# vim nginx.conf 

worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream
    sendfile        on;
    keepalive_timeout  65;
    kafka;
    #kafka地址信息
    kafka_broker_list laosan1:9092;
    server {
        listen       80;
        server_name  localhost;
		#------------kafka相关配置开始------------
        location = /kafka/action {
				kafka_topic topic_laosan;
		}
		#------------kafka相关配置结束------------
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

重启nginx

• 查看nginx是否启动

[root@10 conf]# ps -ef|grep nginx
root     13118     1  0 22:40 ?        00:00:00 nginx: master process ./nginx
nobody   13119 13118  0 22:40 ?        00:00:00 nginx: worker process
root     17994  4800  0 23:26 pts/2    00:00:00 grep --color=auto nginx

• 若启动则停止

[root@10 conf]# cd /usr/local/nginx/sbin
[root@10 sbin]# ./nginx -s stop
[root@10 sbin]# ps -ef|grep nginx
root     18008  4800  0 23:27 pts/2    00:00:00 grep --color=auto nginx

• 重新启动nginx，且指定配置文件路径

[root@10 sbin]# /usr/local/nginx/sbin/nginx -c /home/laosan/software/nginx-1.17.8/conf/nginx.conf
[root@10 sbin]# ps -ef|grep nginx
root     18011     1  0 23:27 ?        00:00:00 nginx: master process /usr/local/nginx/sbin/nginx -c /home/laosan/software/nginx-1.17.8/conf/nginx.conf
nobody   18012 18011  0 23:27 ?        00:00:00 nginx: worker process
root     18017  4800  0 23:27 pts/2    00:00:00 grep --color=auto nginx

测试

• 输入测试路径进行测试

[root@10 sbin]# curl localhost/kafka/action -d "hello laosan"

• 消费者可以看到消费过来的消息

[root@10 ~]# kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic_laosan
hello laosan