log_format backend '$http_x_forwarded_for [$time_local] '
'"$host" "$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"'
[luohui@BJ-huasuan-h-web-07 ~]$ cat /home/luohui/logstash-5.0.0/etc/logstash-nginx.conf
input {
file {
path => ["/home/data/logs/access.log"]
type => "nginx_access"
}
}
output {
if [type] == "nginx_access"{
redis {
host => ["192.168.10.100:6379"]
data_type =>"list"
key => "nginx"
}
}
}
| |
[root@mail etc]# cat logstash_nginx.conf
input {
redis {
host => "192.168.10.100"
port => 6379
data_type => "list"
key => "nginx"
}
}
filter {
grok {
match =>
{"message" => "%{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] %{NOTSPACE:http_name} \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes:float}|-) %{QS:referrer} %{QS:agent}"
}
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
target => "geoip"
database => "/test/logstash-5.0.0/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
output {
elasticsearch {
action => "index"
hosts =>"192.168.10.100:9200"
index => "logstash-nginx-%{+yyyy.MM.dd}"
}
}
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
[root@controller logstash-5.0.0]# cd vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.0.2/patterns[root@controller patterns]# cat nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} - %{NGUSER:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes:float}|-) %{QS:referrer} %{QS:agent} %{NOTSPACE:http_x_forwarded_for} %{NUMBER:request_time:float}
input {
redis {
host => "192.168.10.100"
port => 6379
data_type => "list"
key => "nginx"
}
}
filter {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
target => "geoip"
database => "/test/logstash-5.0.0/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
output {
stdout{codec=>rubydebug}
elasticsearch {
action => "index"
hosts => "192.168.63.235:9200"
index => "logstash-nginx-%{+yyyy.MM.dd}"
}
}
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
access_log /etc/nginx/logs/access_nginx.json json;
[root@controller logstash-5.0.0]# cat etc/nginx_json.conf
input {
file { #从nginx日志读入
type => "nginx-access"
path => "/etc/nginx/logs/access_nginx.json"
start_position => "beginning"
codec => "json" #这里指定codec格式为json
}
}
filter {
if [type] == "nginx-access"{
geoip {
source => "clientip"
target => "geoip"
database => "/test/logstash-5.0.0/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
}
}
output {
if [type] == "nginx-access" {
stdout{codec=>rubydebug}
elasticsearch {
action => "index"
hosts => "192.168.63.235:9200"
index => "mysql-slow-%{+yyyy.MM.dd}"
}
}
}