ELK收集NGINX日志

参考
首先保证nginx启动

[root@localhost ~]# yum -y install nginx
[root@localhost ~]# systemctl start nginx
[root@localhost ~]# netstat -ntlp | grep 80
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      21813/nginx: master
tcp6       0      0 :::80 

1.编辑正则
logstash grok 内置正则：https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns

[root@localhost patterns]# pwd
/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns
[root@localhost patterns]# cat nginx_access
URIPARAM1 [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
NGINXACCESS %{IPORHOST:client_ip} (%{USER:ident}|- ) (%{USER:auth}|-) \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} (%{NOTSPACE:request}|-)(?: HTTP/%{NUMBER:http_version})?|-)" %{NUMBER:status} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" "%{GREEDYDATA:agent}"

2.添加logstash的nginx 日志收集配置文件

[root@localhost conf.d]# cat /etc/logstash/conf.d/nginx.conf
input {
        file {
                path => "/var/log/nginx/access.log"
                type => "nginx-log"
                start_position => "beginning"
        }

}
filter {
        grok {
                match => {"message" => "%{NGINXACCESS}"}
        }
}
output {
        elasticsearch {
                hosts => "192.168.88.8:9200"
                index => "nginx_log-%{+YYYY.MM.dd}"
        }
}

3.权限

[root@localhost conf.d]# chmod -R 777 /var/log

4.管道

[root@localhost logstash]# cat /etc/logstash/pipelines.yml | grep -v "^#" | sed '/^$/d'
- pipeline.id: msg
  path.config: "/etc/logstash/conf.d/messages.conf"
- pipeline.id: nginx
  path.config: "/etc/logstash/conf.d/nginx.conf"

5.重启

[root@localhost logstash]# systemctl restart logstash

6.查看日志

[root@localhost logstash]# tailf /var/log/logstash/logstash-plain.log
[2020-04-10T10:19:11,925][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//192.168.88.8:9200"]}
[2020-04-10T10:19:11,931][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2020-04-10T10:19:12,463][INFO ][logstash.inputs.file     ] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/var/lib/logstash/plugins/inputs/file/.sincedb_452905a167cf4509fd08acb964fdb20c", :path=>["/var/log/messages"]}
[2020-04-10T10:19:12,473][INFO ][logstash.inputs.file     ] No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/var/lib/logstash/plugins/inputs/file/.sincedb_d883144359d3b4f516b37dba51fab2a2", :path=>["/var/log/nginx/access.log"]}
[2020-04-10T10:19:12,538][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"nginx", :thread=>"#<Thread:0x44d86c32 run>"}
[2020-04-10T10:19:12,546][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"msg", :thread=>"#<Thread:0x2887d200 run>"}
[2020-04-10T10:19:12,609][INFO ][logstash.agent           ] Pipelines running {:count=>2, :running_pipelines=>[:msg, :nginx], :non_running_pipelines=>[]}
[2020-04-10T10:19:12,670][INFO ][filewatch.observingtail  ] START, creating Discoverer, Watch with file and sincedb collections
[2020-04-10T10:19:12,671][INFO ][filewatch.observingtail  ] START, creating Discoverer, Watch with file and sincedb collections
[2020-04-10T10:19:13,431][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

7.查看端口

[root@localhost logstash]# netstat -ntlp | grep 9600
tcp6       0      0 127.0.0.1:9600          :::*                    LISTEN      36990/java

8.压测一些数据

[root@localhost ~]# yum -y install httpd-tools
[root@localhost ~]# ab -n 100 -c 100 http://192.168.88.88/index.html
This is ApacheBench, Version 2.3 <$Revision: 1430300 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 192.168.88.88 (be patient).....done


Server Software:        nginx/1.16.1
Server Hostname:        192.168.88.88
Server Port:            80

Document Path:          /index.html
Document Length:        4833 bytes

Concurrency Level:      100
Time taken for tests:   0.027 seconds
Complete requests:      100
Failed requests:        0
Write errors:           0
Total transferred:      506800 bytes
HTML transferred:       483300 bytes
Requests per second:    3765.34 [#/sec] (mean)
Time per request:       26.558 [ms] (mean)
Time per request:       0.266 [ms] (mean, across all concurrent requests)
Transfer rate:          18635.51 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    7   1.4      6       9
Processing:     5    8   2.5      9      12
Waiting:        0    8   2.7      9      12
Total:         10   15   1.3     15      17

Percentage of the requests served within a certain time (ms)
  50%     15
  66%     16
  75%     16
  80%     16
  90%     17
  95%     17
  98%     17
  99%     17
 100%     17 (longest request)

9.查看索引

[root@localhost ~]# curl -X GET http://192.168.88.8:9200/_cat/indices?v
health status index                uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .kibana_1            c2riR6WeT8-Zv_I-r-_xeQ   1   0          3            0     15.6kb         15.6kb
yellow open   nginx_log-2020.04.10 Wd-yWDgHSbmuYAz-PLfy0A   5   1        700            0    298.5kb        298.5kb
yellow open   msg_log-2020.04.10   _qLj50YKSAiqCCaV63I03w   5   1       1929            0    746.4kb        746.4kb

10.去web端添加图形