elk之filebeat安装配置并使用logstash接收数据

下载:

wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.4.2-x86_64.rpm

安装:

rpm -ivh filebeat-6.4.2-x86_64.rpm

配置:

vim /etc/filebeat/filebeat.yml

filebeat.inputs:

# Each – is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.

– type: log
enabled: true
paths:
– /opt/lampp/logs/access_log
fields:
index_name: new_guanwang_access_master

#—————————– Logstash output ——————————–
output.logstash:
# The Logstash hosts
hosts: [“192.168.3.124:5044”]

重启

/etc/init.d/filebeat restart

logstash配置

input {
beats {
port => “5044”
}
}
filter {
if [fields][index_name] == “tour_alllegend_access” and [fields][index_name] == “new_guanwang_access_master” {
grok {
match => { “message” => “%{COMMONAPACHELOG}” }
}
date {
match => [ “timestamp”, “dd/MMM/YYYY:H:m:s Z” ]
target => “@timestamp”
}
} else {
grok {
match => { “message” => “%{COMBINEDAPACHELOG}” }
}
date {
match => [ “timestamp”, “dd/MMM/YYYY:H:m:s Z” ]
target => “@timestamp”
}
}
}
output {
elasticsearch {
hosts => [“localhost:9200”]
index => “%{[fields][index_name]}-%{+YYYY.MM.dd}”
}
}

 

删除filebeat历史上传记录

rm /var/lib/filebeat/registry

 

停止 删除 启动 tail

/etc/init.d/filebeat stop;rm /var/lib/filebeat/registry;/etc/init.d/filebeat start;sleep 3;tail -f /var/log/filebeat/filebeat -n 100

 

} else if [fields][index_name] == “new_guanwang_access_slave” {
16 grok {
17 match => { “message” => “%{IPORHOST:clientip} %{USER:ident} %{NOTSPACE:auth} \[%{HTTPDATE:timestamp}\] \”(?:%{WORD:verb} %{NOTSPACE:re
quest}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\” %{NUMBER:response} (?:%{NUMBER:bytes}|-) \”%{NOTSPACE:referrer}\” \”%{GREEDYDAT
A:agent}\” \”%{IPORHOST:http_x_forwarded_for}\”” }
18 }
19 date {
20 match => [ “timestamp”, “dd/MMM/YYYY:H:m:s Z” ]
21 target => “@timestamp”
22 }
23 } else {