ELK平台搭建与使用

简介

“ELK”是三个开源项目的首字母缩写，这三个项目分别是：Elasticsearch、Logstash 和 Kibana。

• Elasticsearch是实时全文搜索和分析引擎，提供搜集、分析、存储数据三大功能；是一套开放REST和JAVA API等结构提供高效搜索功能，可扩展的分布式系统。它构建于Apache Lucene搜索引擎库之上。

• Logstash是一个用来搜集、分析、过滤日志的工具。它支持几乎任何类型的日志，包括系统日志、错误日志和自定义应用程序日志。它可以从许多来源接收日志，这些来源包括 syslog、消息传递（例如 RabbitMQ）和JMX，它能够以多种方式输出数据，包括电子邮件、websockets和Elasticsearch。

• Kibana是一个基于Web的图形界面，用于搜索、分析和可视化存储在 Elasticsearch指标中的日志数据。它利用Elasticsearch的REST接口来检索数据，不仅允许用户创建他们自己的数据的定制仪表板视图，还允许他们以特殊的方式查询和过滤数据。
  

docker-compose.yml

version: '2.2'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - esdata:/usr/share/elasticsearch/data
    hostname: elasticsearch
    restart: always
    ports:
      - 9200:9200
      - 9300:9300
  kibana:
    image: docker.elastic.co/kibana/kibana:7.1.1
    container_name: kibana
    environment:
      - elasticsearch.hosts=http://elasticsearch:9200
    hostname: kibana
    depends_on:
      - elasticsearch
    restart: always
    ports:
      - "5601:5601"
    volumes:
      - $PWD/kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml
  logstash:
    image: docker.elastic.co/logstash/logstash:7.1.1
    container_name: logstash
    hostname: logstash
    restart: always
    depends_on:
      - elasticsearch
    volumes:
      - $PWD/logstash/conf.d/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    ports:
      - 9600:9600
      - 5044:5044
      - 4560:4560
volumes:
  esdata:
    driver: local

kibana.yml

#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
server.basePath: "/elk"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"

logstash.conf

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4560
    codec => json_lines
    type => "business"
  }
}
output{
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    action => "index"
    codec => json
    index => "logstash-%{+YYYY.MM.dd}"
    template_name => "business"
  }
  stdout {
    codec => rubydebug
  }
}

nginx代理配置

location /es/ {
	proxy_pass http://IP:9200/;
}

location /elk/ {
	proxy_pass http://IP:5601/;
	rewrite ^/elk/(.*)$ /$1 break;
	proxy_set_header  X-Real-IP  $remote_addr;
	proxy_set_header  Host $host:$server_port;
	proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_http_version 1.1;
}

springboot输出日志到logstash

pom.xml

<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>6.0</version>
</dependency>

logback.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
    <include resource="org/springframework/boot/logging/logback/base.xml" />

    <!-- 定义日志的目录 -->
    <property name="LOG_HOME" value="logs"/>
    <!--应用名称-->
    <springProperty name="APP_NAME" scope="context" source="spring.application.name" defaultValue="springBoot"/>

    <property name="LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n"/>


    <!-- 滚动记录文件，先将日志记录到指定文件，当符合某个条件时，将日志记录到其他文件 -->
    <appender name="FILE_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!-- 指定日志文件的名称 -->
        <file>${LOG_HOME}/${APP_NAME}.log</file>
        <encoder>
            <pattern>${LOG_PATTERN}</pattern>
            <charset>utf8</charset>
        </encoder>
        <!--
           当发生滚动时，决定 RollingFileAppender 的行为，涉及文件移动和重命名
           TimeBasedRollingPolicy： 最常用的滚动策略，它根据时间来制定滚动策略，既负责滚动也负责出发滚动。
        -->
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!--路径-->
            <fileNamePattern>${LOG_HOME}/${APP_NAME}-%d{yyyy-MM-dd}-%i.log</fileNamePattern>
            <MaxHistory>30</MaxHistory>
            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>100MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
        </rollingPolicy>
        <layout class="ch.qos.logback.classic.PatternLayout">
            <pattern>${LOG_PATTERN}</pattern>
        </layout>
    </appender>

    <!-- logstash 配置部分 appanme 根据实际情况修改 -->
    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <!--可以访问的logstash日志收集端口-->
        <destination>10.10.9.157:4560</destination>
        <!--<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
           &lt;!&ndash; &lt;!&ndash;自定义字段，区分应用名称&ndash;&gt;
            <customFields>{"appname": "${APP_NAME}"}</customFields>&ndash;&gt;
        </encoder>-->
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <!--自定义日志输出格式-->
                <pattern>
                    <pattern>
                        {
                        "service": "${APP_NAME:-}",
                        "level": "%level",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger",
                        "traceId": "%X{traceId:-}",
                        "message": "%message",
                        "stack_trace": "%exception"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
    </appender>

    <!--指定logger name为包名或类全名 指定级别 additivity设置是否传递到root logger -->
    <logger name="slf4j" level="INFO" additivity="false">
        <appender-ref ref="FILE_LOG"/>
        <appender-ref ref="LOGSTASH"/>
    </logger>
    <!--slf4j2包下的类在ERROR级别时候传递到root logger中-->
    <logger name="slf4j2" level="ERROR"/>

    <root level="INFO">
        <appender-ref ref="FILE_LOG"/>
        <appender-ref ref="LOGSTASH"/>
    </root>
</configuration>