基于ELK+Kafka的日志搜集框架

Kafka

kafka的配置与运行方式百度上很多，此处不再赘述。

ElasticSearch

ElasticSearch只需要配置一个跨域即可直接启动运行，在elasticsearch.yml文件结尾添加如下配置：

http.cors.enabled: true
http.cors.allow-origin: "*"

然后双击/bin目录下的 elasticsearch.bat 文件即可启动。

Kibana

Kibana配置文件在/config目录下的 kibana.yml 文件，其所需配置如下：

# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601

# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
# The default is 'localhost', which usually means remote machines will not be able to connect.
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "localhost"

# The URLs of the Elasticsearch instances to use for all your queries.
elasticsearch.hosts: ["http://localhost:9200"]

# Specifies locale to be used for all localizable strings, dates and number formats.
# Supported languages are the following: English - en , by default , Chinese - zh-CN .
#i18n.locale: "en"
i18n.locale: "zh-CN"

然后双击/bin目录下的 kibana.bat 文件即可启动。

Logstash配置

Logstash的配置是重点，我使用的配置如下：

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  kafka {	#代表数据来源于kafka
    bootstrap_servers => "127.0.0.1:9092"	#kafka服务地址
	topics => ["service"]					#logstash要订阅的主题
	codec => plain							#plain代表输入的是纯文本，此外还有json和multiline选项
	group_id => "servicegroup"				#指定kafka消费者组
	auto_offset_reset => "latest"			#消费者读取数据策略，有latest和earliest选项
	consumer_threads => 5					#读取数据时可由几个线程并行读取
	decorate_events => true					#可向事件添加Kafka元数据，比如主题、消息大小的选项
  }
}

filter {
  grok {
    match => {
    #将输入的message字段分割成timestamp、loglevel、pid、thread、class、info几个小字段
	  "message" => "%{TIMESTAMP_ISO8601:timestamp} *%{LOGLEVEL:loglevel} %{NUMBER:pid} --- \[(?<thread>.+)\] (?<class>.+) : (?<info>.+)"
	}
  }
  date {
  	#将分割出的timestamp字段格式化后填充至@timestamp时间戳字段，目的是让kibana中使用的时间戳是程序执行的时间而不是日志传输到logstash后的时间
    match => ["timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"]
	target => "@timestamp"
  }
  mutate {
    remove_field => ["message"]		#message字段已被分割不再需要，将其删除
	gsub => ["class"," ",""]		#将class中的空格删除
  }
  if ![info] {
	drop {}							#若info字段无内容则丢弃此日志
  }
}

output {
  elasticsearch {
    hosts => ["http://127.0.0.1:9200"]		#ES服务地址
    index => "service"						#要传输到的ES的索引名
	document_type => "logs"					#文件类型
    #user => "elastic"
    #password => "changeme"
  }
}

在input中使用kafka插件的具体属性可以看这个链接：https://segmentfault.com/a/1190000016595992

Logback配置

之前我做的是一个基于springboot的项目，使用Logback日志框架输出日志到kafka和控制台，其所需依赖如下：

        <!-- logback-kafka-appender -->
        <dependency>
            <groupId>com.github.danielwegener</groupId>
            <artifactId>logback-kafka-appender</artifactId>
            <version>0.2.0-RC2</version>
        </dependency>

        <!-- slf4j-api -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
        </dependency>

        <!-- logback-core -->
        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-core</artifactId>
        </dependency>

logback-spring.xml配置文件如下：

<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
    <contextName>logback</contextName>

    <substitutionProperty name="logging.pattern.console"
                          value="%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>

    <substitutionProperty name="logging.pattern.kafka"
                          value="%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>

    <conversionRule conversionWord="clr"
                    converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex"
                    converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx"
                    converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>

    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <layout class="ch.qos.logback.classic.PatternLayout">
            <pattern>
                ${logging.pattern.console}
            </pattern>
        </layout>
    </appender>

    <appender name="kafka" class="com.github.danielwegener.logback.kafka.KafkaAppender">
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>DEBUG</level>
        </filter>
        <encoder>
            <Pattern>
                ${logging.pattern.kafka}
            </Pattern>
        </encoder>
        <topic>joyojservice</topic>
        <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy"/>
        <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy"/>
        <producerConfig>bootstrap.servers=127.0.0.1:9092</producerConfig>
        <producerConfig>retries=1</producerConfig>
        <producerConfig>batch-size=16384</producerConfig>
        <producerConfig>buffer-memory=33554432</producerConfig>
        <producerConfig>properties.max.request.size==2097152</producerConfig>
    </appender>

    <logger name="com.lpc" level="TRACE" additivity="true">
    </logger>

    <!-- 配置hibernate的SQL语句日志到kafka -->
    <logger name="org.hibernate.SQL" additivity="false" level="DEBUG" >
        <appender-ref ref="kafka"/>
    </logger>

    <root level="INFO">
        <appender-ref ref="kafka"/>
        <appender-ref ref="console"/>
    </root>
</configuration>

至此 ELK+Kafka的日志搜集框架搭建完成，在Kibana的控制台界面创建一个叫service的索引：

然后在索引模式界面创建新的索引模式：

点击下一步，在时间筛选字段选择@timestamp：

点击创建索引模式后在Discover页面即可选择该索引模式以查看日志记录。

ps：注意在启动各个组件的时候ES和Kafka是最先要启动的，Logstash和Kibana是最后启动。