1. Download
#wget https://download.elastic.co/logstash/logstash/logstash-2.3.4.tar.gz
#tar -xzf logstash-2.3.4.tar.gz
#cd logstash-2.3.4
#./bin/logstash-plugin install logstash-output-webhdfs
Error
Installing logstash-output-webhdfs
Error Bundler::InstallError, retrying 1/10
An error occurred while installing snappy-jars (1.1.0.1.2), and Bundler cannot continue.
Make sure that `gem install snappy-jars -v '1.1.0.1.2'` succeeds before bundling.
WARNING: SSLSocket#session= is not supported
redis-hdfs.conf
input {
redis {
data_type => "list"
key => "logstash"
host => "192.168.0.137"
port => 6379
threads => 3
}
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
mutate {
remove_field => ["@host"]
}
}
output {
stdout{
codec => rubydebug
}
webhdfs {
host => "192.168.0.131" # (required)i
port => "50070"
path => "/unichat/logstash/dt=%{+YYYY-MM-dd}/logstash-%{+HH}.log" # (required)
user => "root" # (required)
codec => line { format => "%{message}" }
}
}
I want to insert redis queue a json value
rpush logstash '{"userID": 1, "dynamicID": "abcddee","event": 1,
"time": "2016-05-09 12:32:22"}'
And I want to the json data write to hdfs like
1,wera223adfaadf,2,2016-05-09 12:34:34
configure output
webhdfs {
host => "192.168.0.131" # (required)i
port => "50070"
path => "/unichat/logstash/%{+YYYY-MM-dd}.log" # (required)
user => "root" # (required)
codec => line { format => "%{userID},%{dynamicID},%{event},%{time}" }
}