ELK-把nginx日志写入kibana和logstash

一 配置nginx

[root@linux-node3 ~]# cd /usr/local/src/
[root@linux-node3 src]# ls
elasticsearch-head elasticsearch-head.tar.gz nginx-1.10.3.tar.gz
[root@linux-node3 src]# tar xvf nginx-1.10.3.tar.gz
[root@linux-node3 src]# cd nginx-1.10.3
[root@linux-node3 nginx-1.10.3]#yum install pcre openssl openssl-devel zlib zlib-devel pcre-devel –y
[root@linux-node3 nginx-1.10.3]# yum -y install gcc gcc-c++ autoconf automake make
[root@linux-node3 nginx-1.10.3]# ./configure --prefix=/usr/local/nginx --with-http_sub_module --with-http_ssl_module
[root@linux-node3 nginx-1.10.3]# make
[root@linux-node3 nginx-1.10.3]# make install
[root@linux-node3 nginx-1.10.3]# cd /usr/local/nginx/
[root@linux-node3 nginx]# ll
total 0
drwxr-xr-x 2 root root 333 Aug 19 15:47 conf
drwxr-xr-x 2 root root 40 Aug 19 15:47 html
drwxr-xr-x 2 root root 6 Aug 19 15:47 logs
drwxr-xr-x 2 root root 19 Aug 19 15:47 sbin

改kibana监听地址，不让外网或者随便人访问。
[root@linux-node3 nginx]# vim /etc/kibana/kibana.yml
server.host: “127.0.0.1”
[root@linux-node3 nginx]# systemctl restart kibana
[root@linux-node3 nginx]# ss –lnt
LISTEN 0 128 127.0.0.1:5601
[root@linux-node3 nginx]# mkdir /usr/local/nginx/conf/conf.d/
[root@linux-node3 nginx]# vim conf/nginx.conf
user nginx;
worker_processes auto;
include /usr/local/nginx/conf/conf.d/*.conf;

[root@linux-node3 nginx]# useradd -s /sbin/nologin -M nginx
[root@linux-node3 nginx]# /usr/local/nginx/sbin/nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful

[root@linux-node3 nginx]# vim /usr/local/nginx/conf/conf.d/kibana18.conf
upstream kibana_server {
server 127.0.0.1:5601 weight=1 max_fails=3 fail_timeout=60;
}

server {
listen 80;
server_name www.kibana18.com;
location / {
proxy_pass http://kibana_server;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection ‘upgrade’;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}

[root@linux-node3 nginx]# /usr/local/nginx/sbin/nginx –t
[root@linux-node3 nginx]# /usr/local/nginx/sbin/nginx

添加域名
C:\Windows\System32\drivers\etc
10.0.0.17 www.kibana18.com;

增加认证
[root@linux-node3 nginx]# yum install httpd-tools –y
[root@linux-node3 nginx]# htpasswd -bc /usr/local/nginx/htppass.txt kibana 123456
Adding password for user kibana
[root@linux-node3 nginx]# chown nginx.nginx /usr/local/nginx/ -R

[root@linux-node3 nginx]# vim /usr/local/nginx/conf/conf.d/kibana18.conf
upstream kibana_server {
server 127.0.0.1:5601 weight=1 max_fails=3 fail_timeout=60;
}

server {
listen 80;
server_name www.kibana18.com;
auth_basic “Restricted Access”;
auth_basic_user_file /usr/local/nginx/htppass.txt;
location / {
proxy_pass http://kibana_server;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection ‘upgrade’;
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}
[root@linux-node3 nginx]# /usr/local/nginx/sbin/nginx –t
[root@linux-node3 nginx]# /usr/local/nginx/sbin/nginx -s reload

二 nginx 日志转json并收集

在第一台机器上装logstash
[root@linux-node3 ~]# cd /usr/local/src/
[root@linux-node3 src]# yum install -y logstash-5.6.5.rpm
[root@linux-node3 src]# /usr/local/nginx/sbin/nginx -s stop
[root@linux-node3 src]# vim /usr/local/nginx/conf/nginx.conf —改日志格式

http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';
    log_format access_json '{"@timestamp":"$time_iso8601",'
        '"host":"$server_addr",'
        '"clientip":"$remote_addr",'
        '"size":$body_bytes_sent,'
        '"responsetime":$request_time,'
        '"upstreamtime":"$upstream_response_time",'
        '"upstreamhost":"$upstream_addr",'
        '"http_host":"$host",'
        '"url":"$uri",'
        '"domain":"$host",'
        '"xff":"$http_x_forwarded_for",'
        '"referer":"$http_referer",'
        '"status":"$status"}';
access_log  /var/log/nginx/access.log  access_json;

[root@linux-node3 src]# mkdir /var/log/nginx/
[root@linux-node3 src]# chown nginx.nginx /var/log/nginx/ -R
[root@linux-node3 src]# vim /usr/local/nginx/conf/nginx.conf
location / {
root html;
index index.html index.htm;
}

 location /nginxweb {
    root   html;
    index  index.html index.htm;
}

做一个主页文件
[root@linux-node3 src]# cd /usr/local/nginx/html/
[root@linux-node3 html]# mkdir nginxweb
[root@linux-node3 html]# vim nginxweb/index.html
Nginx Web

[root@linux-node3 html]# /usr/local/nginx/sbin/nginx –t
[root@linux-node3 html]# /usr/local/nginx/sbin/nginx

[root@linux-node3 ~]# ll /var/log/nginx/access.log

写个nginx的logstash配置文件
[root@linux-node3 ~]# vim /etc/logstash/conf.d/nginx.conf

input{
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx-access-log-17"
    start_position => "beginning"
stat_interval => "2"
codec => "json"
  }
  file {
    path => "/var/log/messages"
    type => "system-log-17"
    start_position => "beginning"
    stat_interval => "2"
  }
}

output{
  if [type] == "nginx-access-log-17" {
    elasticsearch {
    hosts => ["10.0.0.17:9200"]
    index => "logstash-nginx-accesslog-18-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "system-log-17"{
    elasticsearch {
      hosts => ["10.0.0.18:9200"]
      index => "logstash-system-log-17-%{+YYYY.MM.dd}"
    }
  }
}~

[root@linux-node3 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf –t

[root@linux-node3 ~]# chmod 644 /var/log/messages
[root@linux-node3 ~]# systemctl restart logstash
到elasticesearsh-head插件上去看看有没有收到数据

把17的系统日志和niginx日志加入到kibana