Kafka
kafka的配置与运行方式百度上很多,此处不再赘述。
ElasticSearch
ElasticSearch只需要配置一个跨域即可直接启动运行,在elasticsearch.yml文件结尾添加如下配置:
http.cors.enabled: true
http.cors.allow-origin: "*"
然后双击/bin目录下的 elasticsearch.bat 文件即可启动。
Kibana
Kibana配置文件在/config目录下的 kibana.yml 文件,其所需配置如下:
# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601
# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
# The default is 'localhost', which usually means remote machines will not be able to connect.
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "localhost"
# The URLs of the Elasticsearch instances to use for all your queries.
elasticsearch.hosts: ["http://localhost:9200"]
# Specifies locale to be used for all localizable strings, dates and number formats.
# Supported languages are the following: English - en , by default , Chinese - zh-CN .
#i18n.locale: "en"
i18n.locale: "zh-CN"
然后双击/bin目录下的 kibana.bat 文件即可启动。
Logstash配置
Logstash的配置是重点,我使用的配置如下:
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.
input {
kafka { #代表数据来源于kafka
bootstrap_servers => "127.0.0.1:9092" #kafka服务地址
topics => ["service"] #logstash要订阅的主题
codec => plain #plain代表输入的是纯文本,此外还有json和multiline选项
group_id => "servicegroup" #指定kafka消费者组
auto_offset_reset => "latest" #消费者读取数据策略,有latest和earliest选项
consumer_threads => 5 #读取数据时可由几个线程并行读取
decorate_events => true #可向事件添加Kafka元数据,比如主题、消息大小的选项
}
}
filter {
grok {
match => {
#将输入的message字段分割成timestamp、loglevel、pid、thread、class、info几个小字段
"message" => "%{TIMESTAMP_ISO8601:timestamp} *%{LOGLEVEL:loglevel} %{NUMBER:pid} --- \[(?<thread>.+)\] (?<class>.+) : (?<info>.+)"
}
}
date {
#将分割出的timestamp字段格式化后填充至@timestamp时间戳字段,目的是让kibana中使用的时间戳是程序执行的时间而不是日志传输到logstash后的时间
match => ["timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"]
target => "@timestamp"
}
mutate {
remove_field => ["message"] #message字段已被分割不再需要,将其删除
gsub => ["class"," ",""] #将class中的空格删除
}
if ![info] {
drop {} #若info字段无内容则丢弃此日志
}
}
output {
elasticsearch {
hosts => ["http://127.0.0.1:9200"] #ES服务地址
index => "service" #要传输到的ES的索引名
document_type => "logs" #文件类型
#user => "elastic"
#password => "changeme"
}
}
在input中使用kafka插件的具体属性可以看这个链接:https://segmentfault.com/a/1190000016595992
Logback配置
之前我做的是一个基于springboot的项目,使用Logback日志框架输出日志到kafka和控制台,其所需依赖如下:
<!-- logback-kafka-appender -->
<dependency>
<groupId>com.github.danielwegener</groupId>
<artifactId>logback-kafka-appender</artifactId>
<version>0.2.0-RC2</version>
</dependency>
<!-- slf4j-api -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<!-- logback-core -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</dependency>
logback-spring.xml配置文件如下:
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<contextName>logback</contextName>
<substitutionProperty name="logging.pattern.console"
value="%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>
<substitutionProperty name="logging.pattern.kafka"
value="%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>
<conversionRule conversionWord="clr"
converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
<conversionRule conversionWord="wex"
converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
<conversionRule conversionWord="wEx"
converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>
${logging.pattern.console}
</pattern>
</layout>
</appender>
<appender name="kafka" class="com.github.danielwegener.logback.kafka.KafkaAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<encoder>
<Pattern>
${logging.pattern.kafka}
</Pattern>
</encoder>
<topic>joyojservice</topic>
<keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy"/>
<deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy"/>
<producerConfig>bootstrap.servers=127.0.0.1:9092</producerConfig>
<producerConfig>retries=1</producerConfig>
<producerConfig>batch-size=16384</producerConfig>
<producerConfig>buffer-memory=33554432</producerConfig>
<producerConfig>properties.max.request.size==2097152</producerConfig>
</appender>
<logger name="com.lpc" level="TRACE" additivity="true">
</logger>
<!-- 配置hibernate的SQL语句日志到kafka -->
<logger name="org.hibernate.SQL" additivity="false" level="DEBUG" >
<appender-ref ref="kafka"/>
</logger>
<root level="INFO">
<appender-ref ref="kafka"/>
<appender-ref ref="console"/>
</root>
</configuration>
至此 ELK+Kafka的日志搜集框架搭建完成,在Kibana的控制台界面创建一个叫service的索引:
然后在索引模式界面创建新的索引模式:
点击下一步,在时间筛选字段选择@timestamp:
点击创建索引模式后在Discover页面即可选择该索引模式以查看日志记录。
ps:注意在启动各个组件的时候ES和Kafka是最先要启动的,Logstash和Kibana是最后启动。