Spring boot 日志直接推送到elasticsearch上

前言

在分布式系统中定位问题除了时使用日志的链路追踪外，一般还配合一些搜索引擎使用，使用过几种方案后最后跟大家分享下简单实用的方案，Spring boot 日志+elasticsearch

核心依赖

        <!--		logback直接转elasticsearch-->
        <dependency>
            <groupId>com.internetitem</groupId>
            <artifactId>logback-elasticsearch-appender</artifactId>
            <version>1.6</version>
        </dependency>

elasticsearch配置文件

1.url格式如下

<url>http://账号:密码@地址:端口/_bulk</url>

2.index索引前缀 “xxx”

<index>xxx_%date{yyyy-MM-dd}.log</index>

3.maxMessageSize参数数据内容最大值，本文未使用（默认值-1全部数据接收）如下

<maxMessageSize>200</maxMessageSize>

4.BasicAuthentication.java 重写该类用来鉴权 （如果url参数没有使用账号密码，或者账号密码中带有特殊字符的如 #号的）

原类 userInfo = 账号:密码（替换对应字符串）

package com.internetitem.logback.elasticsearch.config;

import com.internetitem.logback.elasticsearch.util.Base64;

import java.net.HttpURLConnection;

public class BasicAuthentication implements Authentication {
    public void addAuth(HttpURLConnection urlConnection, String body) {
        String userInfo = urlConnection.getURL().getUserInfo();
        if (userInfo != null) {
            String basicAuth = "Basic " + Base64.encode(userInfo.getBytes());
            urlConnection.setRequestProperty("Authorization", basicAuth);
        }
    }
}

4.完整配置

<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds">
    <!-- 自定义参数监听 -->
    <property name="LOG_HOME" value="/app/log" />

    <!-- 彩色日志依赖的渲染类 -->
    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex"
                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx"
                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
    <!-- 彩色日志格式 -->
    <property name="CONSOLE_LOG_PATTERN"
              value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
    <!-- 控制台输出 -->
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <pattern>${CONSOLE_LOG_PATTERN}</pattern>
            <charset>utf8</charset>
        </encoder>
    </appender>


    <!-- 按照每天生成日志文件 -->
    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <!--日志文件输出的文件名 -->
            <FileNamePattern>${LOG_HOME}/${spring.application.name}.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
            <maxFileSize>10MB</maxFileSize>
            <totalSizeCap>1GB</totalSizeCap>
            <!--日志文件保留天数 -->
            <MaxHistory>30</MaxHistory>
        </rollingPolicy>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [${PID:-} - %thread] %-5level %logger:%L - %msg%n</pattern>
            <charset>UTF-8</charset>
        </encoder>
    </appender>

    <!-- 异步输出 -->
    <appender name="ASYNC-INFO" class="ch.qos.logback.classic.AsyncAppender">
        <!-- 不丢失日志.默认的,如果队列的80%已满,则会丢弃TRACT、DEBUG、INFO级别的日志 -->
        <discardingThreshold>0</discardingThreshold>
        <!-- 更改默认的队列的深度,该值会影响性能.默认值为256 -->
        <queueSize>256</queueSize>
        <!-- 添加附加的appender,最多只能添加一个 -->
        <appender-ref ref="FILE"/>
    </appender>

    <appender name="ELASTIC" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
        <url>http://账号:密码@地址:端口/_bulk</url>
        <index>xxx_%date{yyyy-MM-dd}.log</index>
        <type>类型可选填</type>
        <!--        <loggerName>生成日志名</loggerName> &lt;!&ndash; optional &ndash;&gt;-->
        <errorLoggerName>错误日志名</errorLoggerName> <!-- optional -->
        <connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
        <errorsToStderr>false</errorsToStderr> <!-- optional (default false) -->
        <includeCallerData>false</includeCallerData> <!-- optional (default false) -->
        <logsToStderr>false</logsToStderr> <!-- optional (default false) -->
        <maxQueueSize>104857600</maxQueueSize> <!-- optional (default 104857600) -->
        <maxRetries>3</maxRetries> <!-- optional (default 3) -->
        <readTimeout>30000</readTimeout> <!-- optional (in ms, default 30000) -->
        <sleepTime>250</sleepTime> <!-- optional (in ms, default 250) -->
        <rawJsonMessage>false</rawJsonMessage> <!-- optional (default false) -->
        <includeMdc>false</includeMdc> <!-- optional (default false) -->
        <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication"/> <!-- optional -->
        <properties>
            <property>
                <name>host</name>
                <value>${HOSTNAME}</value>
                <allowEmpty>false</allowEmpty>
            </property>
            <property>
                <name>severity</name>
                <value>%level</value>
            </property>
            <property>
                <name>thread</name>
                <value>%thread</value>
            </property>
            <property>
                <name>stacktrace</name>
                <value>%ex</value>
            </property>
            <property>
                <name>logger</name>
                <value>%logger</value>
            </property>
        </properties>
        <headers>
            <header>
                <name>Content-Type</name>
                <value>application/json</value>
            </header>
        </headers>
    </appender>
    <!-- 异步输出 -->
    <appender name="ASYNC-ELASTIC" class="ch.qos.logback.classic.AsyncAppender">
        <!-- 不丢失日志.默认的,如果队列的80%已满,则会丢弃TRACT、DEBUG、INFO级别的日志 -->
        <discardingThreshold>0</discardingThreshold>
        <!-- 更改默认的队列的深度,该值会影响性能.默认值为256 -->
        <queueSize>256</queueSize>
        <!-- 添加附加的appender,最多只能添加一个 -->
        <appender-ref ref="ELASTIC"/>
    </appender>
    <!-- show parameters for hibernate sql 专为 Hibernate 定制 -->
    <!-- <logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="TRACE" /> -->
    <logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="INFO" />
    <logger name="org.hibernate.type.descriptor.sql.BasicExtractor" level="INFO" />
    <!-- <logger name="org.hibernate.SQL" level="TRACE" /> -->
    <logger name="org.hibernate.SQL" level="INFO" />
    <logger name="org.hibernate.engine.QueryParameters" level="INFO" />
    <logger name="org.hibernate.engine.query.HQLQueryPlan" level="INFO" />

    <!-- MyBatis log configure -->
    <logger name="com.apache.ibatis" level="INFO"/>
    <logger name="org.mybatis.spring" level="INFO"/>
    <logger name="java.sql.Connection" level="INFO"/>
    <logger name="java.sql.Statement" level="INFO"/>
    <logger name="java.sql.PreparedStatement" level="INFO"/>

    <!-- 减少部分debug日志 -->
    <logger name="druid.sql" level="INFO"/>
    <logger name="org.apache.shiro" level="INFO"/>
    <logger name="org.mybatis.spring" level="INFO"/>
    <logger name="org.springframework" level="INFO"/>
    <logger name="org.springframework.context" level="INFO"/>
    <logger name="org.springframework.beans" level="INFO"/>
    <logger name="com.baomidou.mybatisplus" level="INFO"/>
    <logger name="org.apache.ibatis.io" level="INFO"/>
    <logger name="org.apache.velocity" level="INFO"/>
    <logger name="org.eclipse.jetty" level="INFO"/>
    <logger name="io.undertow" level="INFO"/>
    <logger name="org.xnio.nio" level="INFO"/>
    <logger name="org.thymeleaf" level="INFO"/>
    <logger name="springfox.documentation" level="ERROR"/>
    <logger name="org.hibernate" level="ERROR"/>
    <logger name="com.netflix.loadbalancer" level="INFO"/>
    <logger name="com.netflix.hystrix" level="INFO"/>
    <logger name="com.netflix.zuul" level="INFO"/>
    <logger name="de.codecentric" level="INFO"/>
    <!-- cache INFO -->
    <logger name="net.sf.ehcache" level="INFO"/>
    <logger name="org.apache.commons" level="INFO" />
    <logger name="org.springframework.cache" level="INFO"/>
    <!-- cloud -->
    <logger name="org.apache.http" level="INFO"/>
    <logger name="com.netflix.discovery" level="INFO"/>
    <logger name="com.netflix.eureka" level="INFO"/>
    <!-- 减少nacos日志 -->
    <logger name="com.alibaba.nacos" level="ERROR"/>

    <!-- 业务日志 -->
    <Logger name="com.cl.frame" level="INFO"/>
    <Logger name="com.cl.framecore.version" level="INFO"/>
    <Logger name="com.cl.frame.shiro.jwt" level="ERROR"/>

    <!-- 日志输出级别 -->
    <root level="INFO">
        <appender-ref ref="ASYNC-INFO"/>
        <appender-ref ref="STDOUT"/>
        <appender-ref ref="ASYNC-ELASTIC"/>
    </root>

</configuration>

使用效果