Logstash To HDFS & 嵌套json拆解

下载插件：

#git clone  https://github.com/heqin5136/logstash-output-webhdfs-discontinued.git
#ls
logstash-output-webhdfs-discontinued

安装logstash-output-webhdfs插件

#cd logstash-output-webhdfs-discontinued
logstash的bin目录下有个plugin，使用plugin来安装插件
#/opt/logstash/bin/plugin install logstash-output-webhdfs

原始json数据如下：

{"thread":"http-nio-9990-exec-2","message":"{\"key3\":\"v4\",\"key4\":\"v3\"}","@version":"1","tags":["production"],"logger":"org.user_check_service.controller.UserCheckService","@timestamp":"2020-01-12T16:51:02.473Z","type":"Parker","source":"user_check_service","level":"INFO","properties":{"varname":"Parker"},"teamName":"ddd","host":"RUU5J4QN1UUYQBB","path":"node1"}

其中"message":"{\"key3\":\"v4\",\"key4\":\"v3\"}"为嵌套的json

 

logstash配置：

input {
 redis {
  codec => json
  host => "192.168.1.98"
  port => 6379
  key => "logstash"
  data_type => "list"
 }
}

filter {

mutate {
      add_field => { "message1" => "%{[message]}" }
    }
json {
        source => "message"
        remove_field => [ "message" ]
    }

}
output {
    #elasticsearch {
    #index => "log-%{[source]}-%{+YYYY.MM.dd}"
    #hosts => ["192.168.1.98:9200"]
    #}
if [type] == "Parker" {
    webhdfs {
           #workers => 2
           host => "192.168.1.123"
           #standby_host => "standbynamenode"
           port => 50070
           user => "hdfsuser"
           path => "/user/Service-Data1/%{+YYYY}-%{+MM}-%{+dd}/logstash-%{+HH}.log"
           #flush_size => 100
           #idle_flush_time => 10
           #compression => "gzip"
           #retry_interval => 3
           codec => 'json'
           }
}
if [type] != "Parker" {
    webhdfs {
           #workers => 2
           host => "192.168.1.123"
           #standby_host => "standbynamenode"
           port => 50070
           user => "hdfsuser"
           path => "/user/Service-Data/%{+YYYY}-%{+MM}-%{+dd}/logstash-%{+HH}.log"
           #flush_size => 100
           #idle_flush_time => 10
           #compression => "gzip"
           #retry_interval => 3
           codec => 'json'
           }
}
 

user => "hdfsuser" 这个user要注意，是启动start-hadoop-namenode的用户

mutate {
      add_field => { "message1" => "%{[message]}" }
    }

 

写入hdfs的格式：

{
      "teamName" => "ddd",
          "type" => "Parker",
          "tags" => [
        [0] "production"
    ],
          "host" => "RUU5J4QN1UUYQBB",
        "source" => "user_check_service",
    "properties" => {
        "varname" => "Parker"
    },
      "@version" => "1",
        "thread" => "http-nio-9990-exec-5",
          "key5" => "v1",
        "logger" => "org.user_check_service.controller.UserCheckService",
    "@timestamp" => 2020-01-12T17:44:01.390Z,
          "key1" => "v5",
          "path" => "node1",
      "message1" => "{\"key1\":\"v5\",\"key5\":\"v1\"}",
         "level" => "INFO"
}