分布式框架-ELK

es 启动异常

1、Unable to establish loopback connection

解决方法：关闭防火墙

1、启动es、

2、kabana、

3、启动logstash

创建对应要导入数据的config文件（logstash**.conf,注意文件里要严格对应花括号，不能有多余空格)

1、logstash -f 目录（包含多个conf文件，logstash会一起导入））

2、logstash -f 目录/conf文件

input {
 # 我们创建了两个微服务demo 所以建立两个不同的输入，将两个服务的日志分别输入到不同的索引中
  tcp {
    mode => "server"
    host => "0.0.0.0"  # 允许任意主机发送日志
    type => "elk1"      # 设定type以区分每个输入源
    port => 4567      
    codec => json_lines{
        charset => "utf-8"
    }
  }
  tcp {
    mode => "server"
    host => "0.0.0.0"
    type => "elk2"
    port => 4667
    codec => json_lines{
        charset => "utf-8"
    }
  }

}
filter {
  mutate {
    remove_field => [@version]
  }
  json {
    source => "enhancemessage"
    remove_field =>["enhancemessage"]
  }
}

output {
  if [type] == "elk1" {
      if "_jsonparsefailure" not in [tags] {
          elasticsearch {
              action => "index"          # 输出时创建映射
              hosts  => "localhost:9200"   # ElasticSearch 的地址和端口
              index  => "elk1-%{+YYYY.MM.dd}"         # 指定索引名
              codec  => "json"
          }
      }
    
  }
  if [type] == "elk2" {
      if "_jsonparsefailure" not in [tags] {
          elasticsearch {
              action => "index"          #The operation on ES
              hosts  => "localhost:9200"   #ElasticSearch host, can be array.
              index  => "elk2-%{+YYYY.MM.dd}"         #The index to write data to.
              codec  => "json"
          }
      }
  }
}

4、启动项目 (配置logback.xml)

<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
    <include resource="org/springframework/boot/logging/logback/base.xml"/>
    <conversionRule conversionWord="mesWithStack" converterClass="com.example.elk.converter.StackTraceMessageConverter" />
    <conversionRule conversionWord="MDCMes" converterClass="com.example.elk.converter.MDCToJsonConverter" />

    <springProperty scope="context" name="springAppName" source="spring.application.name" />
    <appender name="STASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <springProfile name="local">
            <!-- 负载均衡-->
            <destination>localhost:9999</destination><!--logstash机器ip和端口-->
            <destination>localhost:9998</destination><!--logstash机器ip和端口-->
            <destination>localhost:9997</destination><!--logstash机器ip和端口-->
            <connectionStrategy>
                <roundRobin>
                    <connectionTTL>5 minutes</connectionTTL>
                </roundRobin>
            </connectionStrategy>
        </springProfile>
        <encoder charset="utf-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>UTC</timeZone>
                </timestamp>
                <pattern>
                    <pattern>
                        {
                        "severity": "%level",
                        "service": "${springAppName:-}",
                        "ip": "%X{X-B3-ip:-}",
                        "trace": "%X{X-B3-TraceId:-}",
                        "span": "%X{X-B3-SpanId:-}",
                        "exportable": "%X{X-Span-Export:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "enhancemessage": "%MDCMes",
                        "message":"%mesWithStack"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
    </appender>

    <root level="INFO">
        <appender-ref ref="CONSOLE"/>
        <appender-ref ref="STASH"/>
    </root>
    <logger name="org.springframework" level="INFO"/>
    <logger name="com.example.elk" level="DEBUG"/>
</configuration>

5、pom配置

<dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>4.11</version>
        </dependency>

logback各种convert是配置在PatternLayout这个类的

LogstashCommonFieldNames