SS00013.elasticsearch——|Hadoop&ElasticSearch集中式日志分析系统.v13|——|Elasticsearch.v13|

一、日志分析平台实战

一、Nginx部署

### --- 安装git工具，安装wget下载工具

~~~     # 安装相关工具包：Hadoop02
[root@hadoop02 ~]# yum install wget git -y
[root@hadoop02 ~]# yum install gcc-c++ -y
[root@hadoop02 ~]# yum install gcc zlib zlib-devel openssl openssl-devel pcre pcre-devel -y

### --- 下载nginx源码包并解压nginx版本包

~~~     # 下载nginx版本包
[root@hadoop02 software]# wget -c http://nginx.org/download/nginx-1.17.8.tar.gz
~~~     # 解压nginx版本包
[root@hadoop02 software]# tar -zxvf nginx-1.17.8.tar.gz -C /usr/local/src/

### --- 编译安装nginx服务

~~~     # 编译安装nginx服务：进入nginx源码编译目录
[root@hadoop02 ~]# cd /usr/local/src/nginx-1.17.8/
[root@hadoop02 nginx-1.17.8]# ./configure
[root@hadoop02 nginx-1.17.8]# make && make install

### --- 修改nginx的配置文件

~~~     # 修改nginx配置文件：进入nginx安装目录：/usr/local/nginx/conf
[root@hadoop02 ~]# vim /usr/local/nginx/conf/nginx.conf
~~~     # 第35~37行修改nginx默认端口号：把默认端口修改为8080或者8888：此环境我们不需要修改
 35     server {
 36         listen       80;
 37         server_name  localhost;

### --- 启动nginx服务

~~~     # 检查nginx配置文件是否正确
[root@hadoop02 ~]# /usr/local/nginx/sbin/nginx -t
~~~     # 输出参数：说明正确
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful

~~~     # 启动nginx服务

[root@hadoop02 ~]# /usr/local/nginx/sbin/nginx

### --- 验证nginx服务是否启动
### --- 通过web-UI访问nginx服务：http://hadoop02/

~~~     # 查看nginx进程
[root@hadoop02 ~]# ps -ef | grep nginx
nginx: master process /usr/local/nginx/sbin/nginx
nginx: worker process

### --- 查看nginx日志文件

~~~     # nginx日志文件
[root@hadoop02 ~]# tail -f /usr/local/nginx/logs/access.log 
~~~     # nginx输出的日志参数
115.195.145.230 - - [26/Nov/2021:21:10:07 +0800] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36"

二、修改nginx的日志格式为JSON格式

### --- 修改日志格式为JSON格式

~~~     # 修改nginx配置文件未JSON格式：调整nginx产生的日志为json格式，减少Logstash的开销(虽然使用正则可以方便提取出字段，但是效率不高)
~~~     # 将JSON格式的参数添加到nginx.conf文件中
[root@hadoop02 ~]# vim /usr/local/nginx/conf/nginx.conf
~~~     # 第25~35行：添加如下参数；第37行：取消注释，并修改为json格式
      log_format json '{ "@timestamp": "$time_iso8601", '
           '"remote_addr": "$remote_addr", '
           '"remote_user": "$remote_user", '
           '"body_bytes_sent": "$body_bytes_sent", '
           '"request_time": "$request_time", '
           '"status": "$status", '
           '"request_uri": "$request_uri", '
           '"request_method": "$request_method", '
           '"http_referrer": "$http_referer", '
           '"http_x_forwarded_for": "$http_x_forwarded_for", '
           '"http_user_agent": "$http_user_agent"}'; 
   
    access_log  logs/access.log  json;

### --- 重载配置文件

~~~     # 重载配置文件
[root@hadoop02 ~]# /usr/local/nginx/sbin/nginx -s reload
~~~     # 检查配置文件的正确性
[root@hadoop02 ~]# /usr/local/nginx/sbin/nginx -t

### --- 观察日志文件的格式

~~~     # nginx日志文件
[root@hadoop02 ~]# tail -f /usr/local/nginx/logs/access.log 
~~~     # nginx输出的日志参数：没有定义日志文件JSON格式之前的日志
115.195.145.230 - - [26/Nov/2021:21:10:07 +0800] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36"
~~~     # nginx输出的日志参数：定义日志文件JSON格式的日志：刷新nginx-web-UI页面
{ "@timestamp": "2021-11-26T22:07:58+08:00", "remote_addr": "115.195.145.230", "remote_user": "-", "body_bytes_sent": "0", "request_time": "0.000", "status": "304", "request_uri": "/", "request_method": "GET", "http_referrer": "-", "http_x_forwarded_for": "-", "http_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36"}

附录一：nginx.conf配置文件模板

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    log_format json '{ "@timestamp": "$time_iso8601", '
         '"remote_addr": "$remote_addr", '
         '"remote_user": "$remote_user", '
         '"body_bytes_sent": "$body_bytes_sent", '
         '"request_time": "$request_time", '
         '"status": "$status", '
         '"request_uri": "$request_uri", '
         '"request_method": "$request_method", '
         '"http_referrer": "$http_referer", '
         '"http_x_forwarded_for": "$http_x_forwarded_for", '
         '"http_user_agent": "$http_user_agent"}';

    access_log  logs/access.log  json;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }
    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}