ElasticStack中的filebeat

记录下真实案例

下载filebeat

地址：https://www.elastic.co/cn/downloads/past-releases/filebeat-7-11-1
我是用的7.11.1版本的。

ES集群和Kibana

自备集群，搭建还是比较简单的。网上找一下就可以。

filebeat安装和配置

解压filebeat

tar -xzvf filebeat-7.11.1-linux-x86_64.tar.gz

配置filebeat.yml

filebeat.inputs:
- type: filestream
  enabled: true
  paths:
  	# 日志采集的路径
    - /var/logs/*/*.ndjson
    # 滚动创建的日志也进行采集
    - /var/logs/*/*.ndjson-*
  prospector.scanner.check_interval: 10s
  close.on_state_change.inactive: 5m
  close.on_state_change.renamed: true
  close.on_state_change.removed: true

  # # log rotation 滚动的时候以数字结尾
  rotation.external.strategy.copytruncate:
    suffix_regex: \.\d$

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  # processor之解析message，并且输出到ecsmessage字段中
  - decode_json_fields:
      fields: ["message"]
      process_array: false
      max_depth: 5
      target: "ecsmessage"
      overwrite_keys: true
      add_error_key: true
  # processor之删除无用字段
  # 如果有字段message，则删除message、host、ecs、agent、input字段
  - drop_fields:
      when:
        has_fields: ['message']
      fields: ["message", "host", "ecs", "agent","input"]
      ignore_missing: true

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

setup.template.settings:
  index.number_of_shards: 3

# 给索引打上标签
tags: ["xxx"]

fields:
  env: uat

# 设置kibana面板的地址
setup.kibana:
  host: "x.x.x.x:5601"
# 设置elasticsearch的地址
output.elasticsearch:
  hosts: ["x.x.x.x:9200","x.x.x.x:9200","x.x.x.x:9200"]
  # 自定义索引名称
  index: "rotation-logname-%{+yyyy}"
  bulk_max_size: 50
  worker: 1

# 自定义名称的时候，一定要关闭这2个，否则就不成功
setup.ilm.enabled: false
setup.template.enabled: false

# 开启自身的监控
monitoring.enabled: true

spring-boot项目改造

bootstrap.yml 增加配置

logging:
  config: classpath:logback-spring-ecs.xml
  path: /var/logs
server:
  port: 8766

引入es官网提供的maven

	<dependency>
        <groupId>co.elastic.logging</groupId>
        <artifactId>logback-ecs-encoder</artifactId>
    </dependency>

logback-spring-ecs.xml

<?xml version="1.0" encoding="UTF-8" ?>
<configuration debug="true" scan="true" scanPeriod="60 seconds">
    <include resource="org/springframework/boot/logging/logback/defaults.xml" />
    <include resource="org/springframework/boot/logging/logback/console-appender.xml" />
    <include resource="co/elastic/logging/logback/boot/ecs-file-appender.xml" />
    <include resource="co/elastic/logging/logback/boot/ecs-console-appender.xml" />

    <springProperty scope="context" name="ApplicationName" source="spring.application.name"/>
    <springProperty scope="context" name="ServerPort" source="server.port"/>
    <springProperty scope="context" name="LoggingPath" source="logging.path"/>

    <appender name="InfoToJson" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>INFO</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
        <File>${LoggingPath}/${ApplicationName}_${ServerPort}/info-ecs.ndjson</File>
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <FileNamePattern>${LoggingPath}/${ApplicationName}_${ServerPort}/info-ecs.ndjson-%d{yyyyMMdd}.%i</FileNamePattern>
            <maxFileSize>100mb</maxFileSize>
            <MaxHistory>20</MaxHistory>
            <totalSizeCap>2GB</totalSizeCap>
        </rollingPolicy>
        <encoder class="co.elastic.logging.logback.EcsEncoder">
            <serviceName>${ApplicationName}</serviceName>
            <serviceNodeName>${hostName}</serviceNodeName>
            <includeOrigin>true</includeOrigin>
            <additionalField>
                <key>serverPort</key>
                <value>${ServerPort}</value>
            </additionalField>
        </encoder>
    </appender>

    <appender name="ErrorToJson" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
        <File>${LoggingPath}/${ApplicationName}_${ServerPort}/error-ecs.ndjson</File>
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <FileNamePattern>${LoggingPath}/${ApplicationName}_${ServerPort}/error-ecs.ndjson-%d{yyyyMMdd}.%i</FileNamePattern>
            <maxFileSize>100mb</maxFileSize>
            <MaxHistory>20</MaxHistory>
            <totalSizeCap>2GB</totalSizeCap>
        </rollingPolicy>
        <encoder class="co.elastic.logging.logback.EcsEncoder">
            <serviceName>${ApplicationName}</serviceName>
            <serviceNodeName>${hostName}</serviceNodeName>
            <includeOrigin>true</includeOrigin>
            <additionalField>
                <key>serverPort</key>
                <value>${ServerPort}</value>
            </additionalField>
        </encoder>
    </appender>
    <logger name="cn.xxx" level="INFO">
        <appender-ref ref="InfoToJson" />
        <appender-ref ref="ErrorToJson" />
    </logger>
    <root level="INFO">
        <appender-ref ref="CONSOLE" />
    </root>
</configuration>

es中索引管理，超过30天就删除

PUT _ilm/policy/rotation-*
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "30d",
            "max_size": "500gb",
            "max_docs": 20
          },
          "set_priority": {
            "priority": 100
          }
        }
      }
    }
  }
}

参考地址：https://www.elastic.co/guide/en/beats/filebeat/7.11/filtering-and-enhancing-data.html