下载插件:
#git clone https://github.com/heqin5136/logstash-output-webhdfs-discontinued.git
#ls
logstash-output-webhdfs-discontinued
安装logstash-output-webhdfs插件
#cd logstash-output-webhdfs-discontinued
logstash的bin目录下有个plugin,使用plugin来安装插件
#/opt/logstash/bin/plugin install logstash-output-webhdfs
原始json数据如下:
{"thread":"http-nio-9990-exec-2","message":"{\"key3\":\"v4\",\"key4\":\"v3\"}","@version":"1","tags":["production"],"logger":"org.user_check_service.controller.UserCheckService","@timestamp":"2020-01-12T16:51:02.473Z","type":"Parker","source":"user_check_service","level":"INFO","properties":{"varname":"Parker"},"teamName":"ddd","host":"RUU5J4QN1UUYQBB","path":"node1"}
其中"message":"{\"key3\":\"v4\",\"key4\":\"v3\"}"为嵌套的json
logstash配置:
input {
redis {
codec => json
host => "192.168.1.98"
port => 6379
key => "logstash"
data_type => "list"
}
}filter {
mutate {
add_field => { "message1" => "%{[message]}" }
}
json {
source => "message"
remove_field => [ "message" ]
}}
output {
#elasticsearch {
#index => "log-%{[source]}-%{+YYYY.MM.dd}"
#hosts => ["192.168.1.98:9200"]
#}
if [type] == "Parker" {
webhdfs {
#workers => 2
host => "192.168.1.123"
#standby_host => "standbynamenode"
port => 50070
user => "hdfsuser"
path => "/user/Service-Data1/%{+YYYY}-%{+MM}-%{+dd}/logstash-%{+HH}.log"
#flush_size => 100
#idle_flush_time => 10
#compression => "gzip"
#retry_interval => 3
codec => 'json'
}
}
if [type] != "Parker" {
webhdfs {
#workers => 2
host => "192.168.1.123"
#standby_host => "standbynamenode"
port => 50070
user => "hdfsuser"
path => "/user/Service-Data/%{+YYYY}-%{+MM}-%{+dd}/logstash-%{+HH}.log"
#flush_size => 100
#idle_flush_time => 10
#compression => "gzip"
#retry_interval => 3
codec => 'json'
}
}
user => "hdfsuser" 这个user要注意,是启动start-hadoop-namenode的用户
mutate {
add_field => { "message1" => "%{[message]}" }
}
写入hdfs的格式:
{
"teamName" => "ddd",
"type" => "Parker",
"tags" => [
[0] "production"
],
"host" => "RUU5J4QN1UUYQBB",
"source" => "user_check_service",
"properties" => {
"varname" => "Parker"
},
"@version" => "1",
"thread" => "http-nio-9990-exec-5",
"key5" => "v1",
"logger" => "org.user_check_service.controller.UserCheckService",
"@timestamp" => 2020-01-12T17:44:01.390Z,
"key1" => "v5",
"path" => "node1",
"message1" => "{\"key1\":\"v5\",\"key5\":\"v1\"}",
"level" => "INFO"
}