filebeat读取日志存入elasticsearch

logback配置

    <springProperty scope="context" name="LOG_HOME" source="logging.file.path" defaultValue="/var/logs/test-server"/>
    <springProperty scope="context" name="IOT_LOG_HOME" source="iot.log.path" defaultValue="/var/logs/iot"/>
    <springProperty scope="context" name="LOG_NAME" source="spring.application.name" defaultValue="iot-gateway"/>

    <appender name="IOT_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${IOT_LOG_HOME}/${LOG_NAME}.log</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${IOT_LOG_HOME}/${LOG_NAME}_%d{yyyy-MM-dd}.log</fileNamePattern>
            <maxHistory>5</maxHistory>
        </rollingPolicy>
        <encoder charset="UTF-8">
            <pattern>%msg%n</pattern>
        </encoder>
    </appender>

    <logger name="IotLog" level="info" additivity="false">
        <appender-ref ref="IOT_LOG" />
    </logger>

切面

package com.test.test.iot.gateway.aop.aspect;

import cn.hutool.json.JSONConfig;
import cn.hutool.json.JSONUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;

import java.util.Objects;

/**
 * ClassName AuthorizationAspect
 *
 * @author wang
 * Date 2022/4/11 16:42
 */

@Slf4j(topic = "IotLog")
@Aspect
@Order(2)
@Component
@RequiredArgsConstructor
public class IotLogAspect {


    @Pointcut("@annotation(com.test.test.iot.gateway.aop.annotation.IotLog)")
    public void IotLog() {
        // For Aspect
    }

    @Before("IotLog()")
    public void runBefore(JoinPoint joinPoint) {
        Object arg = joinPoint.getArgs()[0];
        if(Objects.isNull(arg)){
            return;
        }
        printLog(arg);

    }

    private void printLog(Object data) {
        JSONConfig jsonConfig = JSONConfig.create().setIgnoreNullValue(false);
        log.info(JSONUtil.toJsonStr(JSONUtil.parseObj(data, jsonConfig)));
    }

}

filebeat配置

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - D:\\workspace\\test-server\\iot\\test-iot-gateway.log
  json.keys_under_root: true
  fields:
    index: 'iot_data_log'

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 5
setup.kibana:

output.elasticsearch:
  hosts: ["192.168.1.101:9200"]
  enabled: true
  indices:
    - index: "iot_data_log"
      when.equals:
        fields:
          index: "iot_data_log"
processors:
  - drop_fields:
      fields: ["log","input","host","agent","ecs"]

# display filebeat logs
logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat.log
  keepfiles: 7
  permissions: 0644

dejavu es可视化界面

http://192.168.1.101:1358/