#Filebeat日志发送到Kafka
filebeat.inputs:
- type: log
tail_files: true
backoff: "1s"
paths:
- /usr/local/nginx/logs/access.json.log
fields:
type: access
fields_under_root: true
output:
kafka:
hosts: ["xxx.xxx.xxx.xx:9092"]
topic: lianxi
#Logstash读取Kafka
input {
kafka {
bootstrap_servers => "xxx.xxx.xxx.xx:9092"
topics => ["lianxi"]
group_id => "lianxi"
codec => "json"
}
}
filter {
if [type] == "access" {
json {
source => "message"
remove_field => ["message","@version","path","beat","input","log","offset","prospector","source","tags"]
}
}
}
output {
stdout {
codec=>rubydebug
}
}
Kafka查看队列信息
查看Group: ./kafka-consumer-groups.sh --bootstrap-server xxx.xxx.xxx.xx:9092 --list
查看队列:./kafka-consumer-groups.sh --bootstrap-server xxx.xxx.xxx.xx:9092 --group lianxi --describe