docker搭建ELK接入Log

目标

1）通过docker安装ELK

2）创建JAVA项目，将日志推送至logstash，由logstash采集后推送至es，并由kibana展示。

目的

方便快速定位报错信息。

V2目标

1）接入sleuth&zipkin

步骤

1）ELK的安装

tips: 可以一个一个装，也可以直接拉取ELK镜像安装。

一、创建网络环境

打开Windows PowerShell

docker network create esnet

二、安装ES

a）拉取镜像：docker pull logstash:7.6.2

b）启动es：docker run -d --name elasticsearch -p 9200:9200 -p 9300:9300 --network esnet -e “discovery.type=single-node” elasticsearch:7.6.2

c）访问es：http://localhost:9200/

d）修改es账户密码：进入容器，进入es的bin目录下执行 ./elasticsearch-setup-passwords interactive

【我配置的密码是：123456，后面Logstash和kibana都用的此密码】

三、安装Logstash

a)拉取镜像：docker pull logstash:7.6.2

b)启动logstash：docker run -d --name logstash --network esnet -p 9600:9600 -p 5044:5044 --restart always logstash:7.6.2

c）通过docker-desktop进入logstash容器：

d）修改logstash.conf内容如下

e）重启logstash

四、安装Kibana

a）拉取镜像：docker pull kibana:7.6.2

b）启动es：docker run --name kibana --worknet esnet -e ELASTICSEARCH_URL=http://127.0.0.1:9200 -p 5601:5601 -d kibana:7.6.2

c）访问es：http://localhost:5601/

d）修改es账号密码

2）项目搭建

a）创建一个springboot项目

b）pom.xml引入依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.timon</groupId>
    <artifactId>sleuth.zipkin</artifactId>
    <version>1.0-SNAPSHOT</version>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.5.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>


    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-logging</artifactId>
        </dependency>
        <!--lombok-->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.8</version>
        </dependency>

        <!--        <!– sleuth –>-->
<!--        <dependency>-->
<!--            <groupId>org.springframework.cloud</groupId>-->
<!--            <artifactId>spring-cloud-starter-sleuth</artifactId>-->
<!--        </dependency>-->
<!--        <!– zipkin client –>-->
<!--        <dependency>-->
<!--            <groupId>org.springframework.cloud</groupId>-->
<!--            <artifactId>spring-cloud-starter-zipkin</artifactId>-->
<!--        </dependency>-->
        <dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>5.1</version>
            <exclusions>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-core</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
    </dependencies>
</project>

c）配置logback-spring.xml

接入logstash主要参考：STASH 的配置

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <include resource="org/springframework/boot/logging/logback/base.xml" />
    <!-- In order to maintain consistency of the log format (%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{50} [%X{logType}] [%X{requestId}] %m%n) -->
    <!-- Log control module will add 'domainName' and 'requestId' before every event message -->
    <property name="LOG_PATTERN" value="%red(%d{yyyy-MM-dd HH:mm:ss}) [%X{traceId},%X{spanId}] %green([%thread]) %highlight(%-5level) %magenta(%logger{40}) - %msg%n"/>
    <property name="LOG_CHARSET" value="utf-8"/>
    <property name="LOG_FILE" value="E:\\logs\\timonTest.log" />



    <logger name="org.springframework.web" level="INFO"/>
    <logger name="org.springboot.sample" level="TRACE" />
    <!-- Filter unnecessary log (include 3PP log) in global scenario with debug level -->
    <logger name="org.eclipse.jetty.io" level="INFO"/>
    <logger name="org.eclipse.jetty.util" level="INFO"/>
    <logger name="org.eclipse.jetty.server.HttpConnection" level="INFO"/>
    <logger name="org.eclipse.jetty.server.HttpInputOverHTTP" level="INFO"/>
    <logger name="org.eclipse.jetty.server.HttpChannelState" level="INFO"/>
    <logger name="org.eclipse.jetty.server.HttpInput" level="INFO"/>
    <logger name="org.eclipse.jetty.server.handler.ContextHandler" level="INFO"/>
    <logger name="org.eclipse.jetty.server.session" level="INFO"/>
    <logger name="org.eclipse.jetty.server.Server" level="INFO"/>
    <logger name="org.eclipse.jetty.servlet.ServletHandler" level="INFO"/>
    <logger name="org.eclipse.jetty.server.HttpChannel" level="INFO"/>
    <logger name="org.eclipse.jetty.http.HttpGenerator" level="INFO"/>
    <logger name="org.eclipse.jetty.http.HttpParser" level="INFO"/>
    <logger name="org.springframework.data.redis.core.RedisConnectionUtils" level="INFO"/>
    <logger name="org.springframework.integration.redis.util.RedisLockRegistry" level="INFO"/>
    <logger name="org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor" level="INFO"/>
    <logger name="org.springframework.beans.factory.support.DefaultListableBeanFactory" level="INFO"/>
    <logger name="org.springframework.boot.actuate.endpoint.mvc.EndpointHandlerMapping" level="INFO"/>
    <logger name="org.springframework.web.servlet.mvc.method.annotation.HttpEntityMethodProcessor" level="INFO"/>
    <logger name="org.apache.http" level="INFO"/>
    <!--    <logger name="org.springframework.web.servlet.DispatcherServlet" level="INFO"/>-->
    <logger name="io.lettuce.core.protocol" level="INFO"/>
    <logger name="com.alibaba.nacos.client.naming" level="WARN"/>
    <logger name="org.springframework.security.web.util.matcher.AntPathRequestMatcher" level="INFO"/>
    <logger name="org.apache.dubbo.remoting.transport.netty4.NettyClientHandler" level="INFO"/>
    <logger name="org.apache.dubbo.remoting.exchange.support.header.HeartbeatHandler" level="INFO"/>
    <logger name="com.alibaba.dubbo.remoting.exchange.support.header.HeartBeatTask" level="INFO"/>
    <logger name="com.alibaba.dubbo.remoting.exchange.support.header.HeartbeatHandler" level="INFO"/>

    <appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>127.0.0.1:5044</destination>
        <includeCallerData>true</includeCallerData>
        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
            <includeCallerData>true</includeCallerData>
            <providers>
                <timestamp>
                    <timeZone>UTC</timeZone>
                </timestamp>
                <pattern>
                    {
                    "severity": "%-5level",
                    "service": "timonTest",
                    "trace": "timonTest",
                    "span": "%X{spanId}",
                    "thread": "%thread",
                    "class": "%logger{40}",
                    "rest": "%msg"
                    }
                </pattern>
            </providers>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
    </appender>

    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <encoder>
            <pattern>${FILE_LOG_PATTERN}</pattern>
        </encoder>
        <file>${LOG_FILE}</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${LOG_FILE}.%d</fileNamePattern>
        </rollingPolicy>
    </appender>


        <logger name="org.springframework.web" level="INFO"/>
        <logger name="org.springboot.sample" level="INFO" />
        <logger name="com.xwtech" level="DEBUG" />
        <logger name="com.timon" level="INFO" additivity="false">
            <appender-ref ref="CONSOLE"/>
            <appender-ref ref="FILE"/>
            <appender-ref ref="STASH"/>
        </logger>



</configuration>

3）快速使用

a）访问 http://localhost:5601/，输入账号密码：elastic/123456

b）创建索引

d）在discover里即可搜索日志