filebeat使用include_lines时,入库慢的解决方法
在filebeat.yaml中添加以下参数
filebeat.registry_flush: 2s
例子
filebeat.registry_flush: 2s
filebeat.prospectors:
- type: log
enabled: true
paths:
- /data/log/smartgate/*.req_[0-9]*.csv
include_lines: [ '\"503\"\,' ]
ignore_older: 2h
### Multiline options
multiline.pattern: '^\"[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
fields:
type: "xtbg-req-authgate"
output.elasticsearch:
worker: 2
bulk_max_size: 256
pipelines:
- pipeline: xtbg-splog-req-authgate
when.equals:
fields.type: "xtbg-req-authgate"