filebeat 日志采集到 elasticsearch

一、下载与安装

1.下载

https://www.elastic.co/cn/downloads/beats/filebeat

根据自己的需求下载对应的版本

2. 解压

# windows平台直接解压

# linux平台解压：
tar -zxvf filebeat-7.6.1-linux-x86_64.tar.gz

二、修改配置文件

1. 正常日志采集

# ============================== Filebeat inputs ===============================
filebeat.inputs:
# 文本算法类型日志采集
- type: log
  enabled: true
  encoding: UTF-8
  fields:
    algo_type: text 
  paths:
    - /home/script/text/logs/*log*
 
# 图片算法日志采集
- type: log
  enabled: true
  encoding: UTF-8
  fields:
    algo_type: image
  paths:
    - /home/script/image/logs/*log*


# ======================= Elasticsearch template setting =======================
# es索引模板配置
setup.ilm.enabled: false
setup.template.name: "filebeat"
setup.template.pattern: "filebeat-*"
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 1
  index.codec: best_compression

# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
  hosts: ["192.168.19.100:9200"]
  protocol: "http"
  # es账号密码，没有设置的可以不用配置
  username: "root"
  password: "123456"
  # 根据不同的日志类型配置不同的es索引
  indices:
    - index: "filebeat-text-%{+yyyy-MM-dd}"
      when.equals:
        fields:
          algo_type: "text"
    - index: "filebeat-image-%{+yyyy-MM-dd}"
      when.equals:
        fields:
          algo_type: "image"


# ================================= Processors =================================
processors:
  # 将自带的一些字段可以去除
  # - add_host_metadata:
  #     when.not.contains.tags: forwarded
  # - add_cloud_metadata: ~
  # - add_docker_metadata: ~
  # - add_kubernetes_metadata: ~
  # 在这里可以设置要去除的字段
  - drop_fields:
      # when: 可以设置去除的条件
      #   condition
      fields: ["log","host","input","agent","ecs"]
      ignore_missing: false

2. JSON格式日志采集

采集JSON格式数据需要将日志输出JSON格式的日志，如下：

{"start_time": "2020-12-13 10:37:01.072","type": "CsbTest","level": "INFO","message":"数据1","parameter":["6677"]}
{"start_time": "2020-12-13 10:37:01.072","type": "CsbTest","level": "INFO","message":"数据2","parameter":["12121"]}

配置如下：

# ============================== Filebeat inputs ===============================
filebeat.inputs:
# 文本算法类型日志采集
- type: log
  enabled: true
  encoding: UTF-8
  fields:
    algo_type: text 
  paths:
    - /home/script/text/logs/*log*
  json:
    keys_under_root: true
    overwrite_keys: true
    add_error_key: true
 
# 图片算法日志采集
- type: log
  enabled: true
  encoding: UTF-8
  fields:
    algo_type: image
  paths:
    - /home/script/image/logs/*log*
  json:
    keys_under_root: true
    overwrite_keys: true
    add_error_key: true


# ======================= Elasticsearch template setting =======================
# es索引模板配置
setup.ilm.enabled: false
setup.template.name: "filebeat"
setup.template.pattern: "filebeat-*"
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 1
  index.codec: best_compression

# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
  hosts: ["192.168.19.100:9200"]
  protocol: "http"
  # es账号密码，没有设置的可以不用配置
  username: "root"
  password: "123456"
  # 根据不同的日志类型配置不同的es索引
  indices:
    - index: "filebeat-text-%{+yyyy-MM-dd}"
      when.equals:
        fields:
          algo_type: "text"
    - index: "filebeat-image-%{+yyyy-MM-dd}"
      when.equals:
        fields:
          algo_type: "image"


# ================================= Processors =================================
processors:
  # 将自带的一些字段可以去除
  # - add_host_metadata:
  #     when.not.contains.tags: forwarded
  # - add_cloud_metadata: ~
  # - add_docker_metadata: ~
  # - add_kubernetes_metadata: ~
  # 在这里可以设置要去除的字段
  - drop_fields:
      # when: 可以设置去除的条件
      #   condition
      fields: ["log","host","input","agent","ecs"]
      ignore_missing: false

三、启动

# linux中，后台启动， 先进入到安装目录中，将所有标准输出及标准错误输出到/dev/null空设备，即没有任何输出
nohup  ./filebeat  -c  filebeat.yml  -e  >/dev/null  2>&1  &

# windows中启动，先进入到安装目录中
.\filebeat.exe -e -c .\filebeat.yml

• -c：配置文件位置
• -e：关闭日志输出