nginx-kafka的安装使用

Nginx (engine x) 是一个高性能的负载均衡和反向代理web服务器，同时也提供了IMAP/POP3/SMTP服务。

一、nginx的安装

1、下载linux版本安装包，解压

2、安装C语言编译器

3、安装依赖库

yum install -y pcre-devel openssl-devel gcc curl

4、重新编译

这样，nginx就安装到了 /usr/local/nginx 下，源码是在 /usr/local/src/nginx-1.19.8

5、启动nginx

关闭防火墙，访问nginx

二、nginx-kafka插件安装

使用该插件，可以直接将nginx里日志数据，直接写到kafka里

1、上传加压kafka的c客户端源码安装包

2、重新编译

3、上传nginx-kafka插件

4、重新编译nginx

5、修改配置文件

进入到 /usr/local/nginx，修改nginx.conf，

#user  nobody;
worker_processes  2;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    kafka; #跟kafka整合
    kafka_broker_list node-1.51doit.cn:9092 node-2.51doit.cn:9092 node-3.51doit.cn:9092; #指定的kafka地址

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

	location = /log.gif {
	    kafka_topic access19;
	}

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

并配置hosts

6、创建kafka生产者和消费者

7、加载so库

8、启动nginx

9、nginx发送数据