ELK获取nginx日志

log_format json '{"@timestamp":"$time_iso8601",'

'"host":"$server_addr",'

'"clientip":"$remote_addr",'

'"size":$body_bytes_sent,'

'"responsetime":$request_time,'

'"upstreamtime":"$upstream_response_time",'

'"upstreamhost":"$upstream_addr",'

'"http_host":"$host",'

'"url":"$uri",'

'"xff":"$http_x_forwarded_for",'

'"referer":"$http_referer",'

'"agent":"$http_user_agent",'

'"status":"$status"}';

或

log_format json '$remote_addr - $remote_user [$time_local] "$request" '

'$status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for" "$remote_port" '

'"$upstream_addr"';

access_log /var/log/nginx/access.log_json json;

注意：在  $request_time  和  $body_bytes_sent  变量两头没有双引号  " ，这两个数据在 JSON 里应该是数值类型！

本地logstash配置

远端logstash配置

分析IP

[root@controller logstash-5.0.0]#cat etc/nginx_json.conf 

input {

   file {             #从nginx日志读入

     type => "logstash-nginx-%{+YYYY-MM} "

     path =>"/etc/nginx/logs/access.json"

     start_position => "beginning" 

     codec => "json"  #这里指定codec格式为json

   }

}

  

filter {

if [type] == "logstash-nginx-%{+YYYY-MM}" {

grok {

match => {

"message" => "(?<clientip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) \- \- \[%{HTTPDATE:timestamp}\] \"%{WORD:http_method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} (?:%{NUMBER:bytes}|\-) \"(?:%{GREEDYDATA:http_referrer}|\-)\" \"(%{GREEDYDATA:user_agent}|\-)\" \"(?<real_ip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|\-)\" \"(?<remote_port>[0-9]+)\" \"(?<upstream_ip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+\s*[\:|\,]\s*[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+)\""

}

remove_field => ["message", "beat.version", "_type", "_id", "_score"]

}

geoip {

source => "clientip"

target => "geoip"

database =>"/etc/logstash/GeoLite2-City.mmdb"

add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}" ]

add_field => [ "[geoip][coordinates]","%{[geoip][latitude]}" ]

}

date {

match => ["timestamp", "YYYY-MM-dd HH:mm:ss"]

remove_field => ["timestamp"]

}

}

}

  

output {

redis {

host => "192.168.30.220"

port => "6379"

data_type => 'list'

key => 'logstash-nginx-%{+YYYY-MM}'

}

}