springboot+logstash+es+kibana 日志服务

安装es

docker run -d --name es -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" -v /opt/elasticsearch/logs:/usr/share/elasticsearch/logs -v /opt/elasticsearch/data:/usr/share/elasticsearch/data -v /opt/elasticsearch/plugins:/usr/share/elasticsearch/plugins --privileged --network es-net -p 9200:9200 -p 9300:9300 elasticsearch:7.12.0

安装 kibana

   docker run -d  --name kibana -e ELASTICSEARCH_HOSTS=http://es:9200 --network es-net -p 5601:5601 kibana:7.12.0

安装logstash

docker run -d   --name logstash --env TZ=Asia/Shanghai -v /opt/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf -p 5044:5044  --network es-net   logstash:7.12.0

修改ES 访问密码

   ./bin/elasticsearch-setup-passwords interactive

修改kibana

server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://es:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
xpack.security.enabled: true
elasticsearch.username: "elastic"
elasticsearch.password: "*****"

修改logstash

配置文件logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.username: "elastic"
xpack.monitoring.elasticsearch.password: "***"
xpack.monitoring.elasticsearch.hosts: [ "http://es:9200" ]

修改 pipeline一logstash.conf

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"  # 允许任意主机发送日志
    port => 5044 # logstash暴露的端口
    codec => json_lines    # 数据格式
  }
}

filter {
    ruby {
        # 将时间转成毫秒的时间戳
        code => "event.set('createTime',(event.get('@timestamp').to_f.round(3)*1000).to_i)"
    }
    ruby {
        # 设置一个自定义字段'timestamp'[这个字段可自定义]，将logstash自动生成的时间戳中的值加8小时，赋给这个字段
    code => "
          event.set('timestamp', event.get('@timestamp').time.localtime + 8*3600)
          event.set('threadName', event.get('thread_name'))
          event.set('levelValue', event.get('level_value'))
          event.set('loggerName', event.get('logger_name'))
          event.set('callerClassName', event.get('caller_class_name'))
          event.set('callerFileName', event.get('caller_file_name'))
          event.set('callerLineNumber', event.get('caller_line_number'))
          event.set('callerMethodName', event.get('caller_method_name'))
          event.set('stackTrace', event.get('stack_trace'))
        "
    }
    ruby {
        # 将自定义时间字段中的值重新赋给@timestamp
        # code => "event.set('@timestamp',event.get('timestamp'))"
    }
    mutate {
        # 删除自定义字段
        remove_field => ["timestamp","thread_name","level_value","logger_name","HOSTNAME","caller_class_name","caller_file_name","caller_line_number","caller_method_name","stack_trace"]
    }
}

output {
  elasticsearch {
      hosts  => ["http://es:9200"]   # ElasticSearch 的地址和端口
      user   => "elastic"
      password => "*****."
      index  => "%{[app_name]}"         # 指定索引名
     # index  => "application-logs-%{[app_name]}-%{[profiles_active]}-%{+YYY-MM}"         # 指定索引名

      # document_type => "_doc"
      codec  => json
      # 是否使用模板创建索引，在模板中可提前定义索引的字段类型
      manage_template => true
      # 索引模板文件
      template => "/usr/share/logstash/pipeline/mappings/application-log-mapping.json"
      template_name => "application-log"
      # 在logstash重启后，是否使用模板文件覆盖es中已存在的索引模板
      template_overwrite => true
  }
  #stdout {
  #  codec => rubydebug
  #}
  file {
    path => "/usr/share/logstash/pipeline/logs/%{+YYYY-MM-dd}-%{app_name}-%{profiles_active}.log"
  }
}

修改mappings一application-log-mapping.json

{
 "index_patterns": ["application-log*"],
 "order": 1,
 "mappings" : {
   "dynamic_templates" : [{
     "message_field" : {
       "match" : "message",
       "match_mapping_type" : "string",
       "mapping" : {
         "type" : "keyword",
         "index" : true
       }
     }
   }, {
     "string_fields" : {
       "match" : "*",
       "match_mapping_type" : "string",
       "mapping" : {
         "type" : "keyword",
         "index" : true
       }
     }
   }],
   "properties" : {
     "@timestamp": { "type": "date" },
     "@version": { "type": "keyword", "index": true },
     "message": {
       "type": "text",
       "analyzer": "ik_max_word"
     }
   }
 }
}

Springboot (版本2.5.15)

集成logstash依赖

 <dependency>
           <groupId>net.logstash.logback</groupId>
           <artifactId>logstash-logback-encoder</artifactId>
           <version>6.6</version>
       </dependency>

修改 logback.xml

 <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
       <!--配置logStash 服务地址-->
       <destination>logstash IP:5044</destination>
       <!-- 日志输出编码 -->
       <encoder charset="UTF-8"
                class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
           <providers>
               <timestamp>
                   <timeZone>UTC</timeZone>
               </timestamp>
               <pattern>
                   <pattern>
                       {
                       "env": "prod",
                       "service":"crm-log",
                       "timestamp":"%d{yyyy-MM-dd HH:mm:ss.SSS}",
                       "date":"%d{yyyy-MM-dd HH:mm:ss.SSS}",
                       "level":"%level",
                       "thread": "%thread",
                       "logger": "%logger{36}",
                       "msg":"%msg",
                       "exception":"%exception",
                       "app_name":"crm-log",
                       "profiles_active":"prod"
                       }
                   </pattern>
               </pattern>
           </providers>
       </encoder>
   </appender>
   
     <root level="info">
     
       <appender-ref ref="LOGSTASH" />

   </root>

kibana 添加索引

management->stach management->kibana->索引模式->关联相应索引

Analytics->discover->选择相应索引