ELKStack-生产案例项目实战
1、收集ES和apache日志,入redis
input { file { path => "/etc/httpd/logs/access_log" start_position => "beginning" type => "apache-accesslog" } file{ path => "/var/log/elasticsearch/myes.log" type => "es-log" start_position => "beginning" codec => multiline{ pattern => "^\[" negate => true what => "previous" } } } output{ if [type] == "es-log" { redis { host => ["192.168.137.11"] port => 6379 db => 1 data_type => "list" key => "es-log" timeout => 10 } } if [type] == "apache-accesslog" { redis { host => ["192.168.137.11"] port => 6379 db => 1 data_type => "list" key => "apache-accesslog" timeout => 10 } } }
启动/opt/logstash/bin/logstash -f /etc/logstash/conf.d/shipper.conf
2、通过syslog服务端主机,获取所有的客户端主机的syslog和redis中数据,写入ES
input{ syslog { type => "system-syslog" port => 514 } redis { type => "es-log" host => ["192.168.137.11"] port => 6379 db => 1 data_type => "list" key => "es-log" timeout => 10 } redis { type => "apache-accesslog" host => ["192.168.137.11"] port => 6379 db => 1 data_type => "list" key => "apache-accesslog" timeout => 10 } } filter { if [type] == "apache-accesslog" { grok { match => { "message" => "%{COMBINEDAPACHELOG}" } } } } output{ if [type] == "apache-accesslog" { elasticsearch { hosts => ["192.168.137.11:9200"] index => "apache-accesslog-%{+YYYY.MM.dd}" } } if [type] == "es-log" { elasticsearch { hosts => ["192.168.137.11:9200"] index => "es-log-%{+YYYY.MM}" } } if [type] == "system-syslog" { elasticsearch { hosts => ["192.168.137.11:9200"] index => "system-syslog-%{+YYYY.MM}" } } }
启动/opt/logstash/bin/logstash -f /etc/logstash/conf.d/redis-es.conf