1. Filebeat 安装及基本操作;
cd /usr/local/src
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.5.0-linux-x86_64.tar.gz
tar zxvf filebeat-7.5.0-linux-x86.tar.gz -C /usr/local/filebeat
cd /usr/local/filebeat/
vim filebeat.yml
- type: log
enabled: true
paths:
- /var/log/*.log
- /usr/local/nginx/logs/access.log
tags: "test_filebeat"
fields:
server: localhost
type: nginx_access
fields_under_root: true
multiline.pattern: ^\d{4}-\d{1,2}-\d{1,2}
multiline.negate: true
multiline.match: after
clean_inactive: 72h
ignore_older: 48h
output.elasticsearch:
hosts: ["localhost:9200"]
username: "es_username"
password: "{pwd}"
output.logstash:
hosts: ["localhost:5044"]
output.redis:
hosts: ["localhost:6379"]
key: "redis"
setup.kibana:
host: "http://192.168.60.221:5601"
username: "kbn_username"
password: "{pwd}"
logging.level: info
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat.log
./filebeat/filebeat setup --dashboards
nohup ./filebeat -e -c filebeat.yml -d "publish" &
2. Filebeat 配合 Logstash 收集解析日志;
- 实际场景中,Logstash 管道会复杂一些,通常有一个或多个 input、filter 和 output 插件
- 场景:创建一个 Logstash 管道,并且使用 Filebeat 将 nginx 的 access.log 日志作为 input、解析这些日志,然后将解析的数据写入 ElasticSearch
2.1 配置 Filebeat 发送日志行到 Logstash;
vim /usr/local/filebeat/filebeat.yml
filebeat.inputs
- type: log
paths:
- /usr/local/nginx/logs/access.log
output.logstash:
hosts: ["localhost:5044"]
vim /usr/local/logstash/first-pipeline.conf
input { beats { port => 5044 } }
output {
stdout {}
}
/usr/local/logstash/bin/logstash -f /usr/local/logstash/first-pipeline.conf --config.reload.automatic
/usr/local/filebeat/filebeat -e -c /usr/local/filebeat/filebeat.yml -d "publish"
2.2 使用 Grok 过滤器插件解析日志;
- 已经完成 Filebeat 读取日志行,但是日志消息的格式并不理想。想要解析日志消息,以便从日志中创建特定的、命名的字段,需要使用 grok filter 插件
- grok 过滤器插件是 Logstash 中默认可用的几个插件之一,将非结构化日志数据解析为结构化和可查询的数据
vim /usr/local/logstash/first-pipeline.conf
input { beats { port => 5044 } }
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
geoip { source => "clientip" }
}
output {
stdout {}
}
rm -rf /usr/local/filebeat/data/registry/
/usr/local/filebeat/filebeat -e -c /usr/local/filebeat/filebeat.yml -d "publish"
2.3 索引数据至 Elasticsearch;
- 之前的配置中,配置了 Logstash 输出到控制台,现在需要修改输出到 Elasticsearch
vim /usr/local/logstash/first-pipeline.conf
output {
elasticsearch {
hosts => [ "localhost:9200" ]
}
}
rm -rf /usr/local/filebeat/data/registry/
/usr/local/filebeat/filebeat -e -c /usr/local/filebeat/filebeat.yml -d "publish"
curl -X GET 'localhost:9200/[索引名]/_search?pretty&q=response=200'
2.4 项目实例;
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/access.log
fields:
source: xxx-pc
type: nginx_access
fields_under_root: true
- type: log
enable: true
paths:
- /usr/local/nginx/logs/error.log
include_lines: ['error']
fields:
source: xxx-pc
type: nginx_error
fields_under_root: true
multiline.pattern: ^\d{4}/\d{1,2}/\d{1,2}
multiline.negate: true
multiline.match: after
input {
beats { port => 5044 }
}
filter {
if [type] == "nginx_access" {
grok {
match => {
"message" =>[
"%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"
]
}
}
date {
match => ["time","dd/MMM/yyyy:HH:mm:ss Z"]
target => "logdate"
}
ruby{
code => "event.set('logdateunix',event.get('logdate').to_i)"
}
geoip {
source => "client_ip"
}
} else if [type] == "nginx_error" {
grok {
match => [
"message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:,\s{1,}client:\s{1,}(?<client_ip>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:client_ip})?(?:, referrer: \"%{URI:referrer})?",
"message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}%{GREEDYDATA:err_message}"
]
}
date{
match => ["time","yyyy/MM/dd HH:mm:ss"]
target => "logdate"
}
ruby{
code => "event.set('logdateunix',event.get('logdate').to_i)"
}
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "xxxxxxxx"
index => "%{[source]}_%{[type]}_%{+YYYY.MM}"
}
}