filebeat 配置模拟采集日志信息发送到kafka

filebeat.yaml

 

filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - /data/service/bigdata/cloud_platform/log/saas_auth_service-core-device-data-point-kv.log
  fields_under_root: true
#正则拼配
  multiline.pattern: '^Hh'
 #默认为false匹配pattern的行合并到上一行,true不匹配pattern的行合并到上一行 
 multiline.negate: true
#after 或 before，合并到上一行的末尾或开头
  multiline.match: after
  fields:
    log_type: saas_auth
    namespace: rel
    hostname: HY-REL-SAAS01
    log_topic: cloud-platform-saas-auth-log

- type: log
  enabled: true
  paths:
    - /data/service/bigdata/cloud_platform/log/saas_core_service-core-data-point-kv.log
  fields_under_root: true
  multiline.pattern: '^Hh'
  multiline.negate: true
  multiline.match: after
  fields:
    log_type: saas_core
    namespace: rel
    hostname: HY-REL-SAAS01
    log_topic: cloud-platform-saas-core-log

- type: log
  enabled: true
  paths:
    - /data/service/bigdata/cloud_platform/log/saas_access_link-data-point-kv.log
  fields_under_root: true
  multiline.pattern: '^Hh'
  multiline.negate: true
  multiline.match: after
  fields:
    log_type: saas_access
    namespace: rel
    hostname: HY-REL-SAAS01
    log_topic: cloud-platform-saas-access-log

output.kafka:
  partition.round_robin:
    group_events: 1
    reachable_only: true
  enabled: true
  hosts: ["bigdata-yace09:6667"]
  topic: '%{[log_topic]}'
  compression: gzip
  compression_level: 9
  max_message_bytes: 10000000
  version: 0.10.1
  retry.max: 10
  timeout: 180