1.logstash filter component组件的正则表达式要写的足够strong,需要充分考虑字段值为空的情况,比如DB字段,如果不考虑,数据会丢失。
logstash内置了kafka output组件。 file input基本实现了tail的功能
logstash-kafka.conf
input{
exec{
command => "cat /db2fs/opt/log-process/share/db2_test.log"
codec => multiline{
pattern => "?\d\d\d\d-\d\d"
negate => true
what => previous
}
interval=>30
}
#file{
# path => "/db2fs/opt/log-process/share/db2_test.log"
# codec => multiline{
# pattern => "?\d\d\d\d-\d\d"
# negate => true
# what => previous
# }
#}
}
filter {
mutate{
gsub => ['message', "\n", " "]
}
}
output {
## elasticsearch { host => localhost }
kafka {
topic_id=>"stream"
broker_list=>"9.115.42.108:9092"
}
}
2.kafka-elastic .conf
input{
kafka{
topic_id => "stream"
}
}
filter {
grok{
patterns_dir=>"/db2fs/opt/log-process/logstash-1.5.0/conf/patterns"
match=>{"message"=>"%{DB2TIME:timestamp}%{INT:timezone}\s*%{WORD:recordid}\s*LEVEL\s*:%{DATA:level}PID\s*:\s*%{NUMBER:processid}\s*TID\s*:\s*%{NUMBER:threadid}\s*PROC\s*:%{DATA:process}INSTANCE\s*:\s*%{WORD:instance}\s*NODE\s*:\s*%{WORD:node}\s*(?:DB\s*:\s%{DBNAME:dbname}\s*)?(?:APPHDL\s*:\s*%{NOTSPACE:apphdl}\s*)?(?:APPID\s*:\s*%{NOTSPACE:appid}\s*)?(?:AUTHID\s*:\s*%{WORD:authid}\s*)?(?:HOSTNAME\s*:\s*%{NOTSPACE:hostname}\s*)?(?:EDUID\s*:\s*%{NUMBER:eduid}\s*)?(?:EDUNAME\s*:\s*%{DATA:eduname}\s*)?FUNCTION\s*:%{DATA:function}probe\s*:\s*%{NUMBER:probe}\s*%{GREEDYDATA:functionlog}"}
}
date{
match=>["timestamp","YYYY-MM-dd-HH.mm.ss.SSSSSS"]
}
}
output {
stdout {
codec=>rubydebug
}
}
3.pattern.txt
DB2TIME %{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}-%{HOUR:hour}.%{MINUTE:minute}.%{SECOND:second}
logstash内置了kafka output组件。 file input基本实现了tail的功能
logstash-kafka.conf
input{
exec{
command => "cat /db2fs/opt/log-process/share/db2_test.log"
codec => multiline{
pattern => "?\d\d\d\d-\d\d"
negate => true
what => previous
}
interval=>30
}
#file{
# path => "/db2fs/opt/log-process/share/db2_test.log"
# codec => multiline{
# pattern => "?\d\d\d\d-\d\d"
# negate => true
# what => previous
# }
#}
}
filter {
mutate{
gsub => ['message', "\n", " "]
}
}
output {
## elasticsearch { host => localhost }
kafka {
topic_id=>"stream"
broker_list=>"9.115.42.108:9092"
}
}
2.kafka-elastic .conf
input{
kafka{
topic_id => "stream"
}
}
filter {
grok{
patterns_dir=>"/db2fs/opt/log-process/logstash-1.5.0/conf/patterns"
match=>{"message"=>"%{DB2TIME:timestamp}%{INT:timezone}\s*%{WORD:recordid}\s*LEVEL\s*:%{DATA:level}PID\s*:\s*%{NUMBER:processid}\s*TID\s*:\s*%{NUMBER:threadid}\s*PROC\s*:%{DATA:process}INSTANCE\s*:\s*%{WORD:instance}\s*NODE\s*:\s*%{WORD:node}\s*(?:DB\s*:\s%{DBNAME:dbname}\s*)?(?:APPHDL\s*:\s*%{NOTSPACE:apphdl}\s*)?(?:APPID\s*:\s*%{NOTSPACE:appid}\s*)?(?:AUTHID\s*:\s*%{WORD:authid}\s*)?(?:HOSTNAME\s*:\s*%{NOTSPACE:hostname}\s*)?(?:EDUID\s*:\s*%{NUMBER:eduid}\s*)?(?:EDUNAME\s*:\s*%{DATA:eduname}\s*)?FUNCTION\s*:%{DATA:function}probe\s*:\s*%{NUMBER:probe}\s*%{GREEDYDATA:functionlog}"}
}
date{
match=>["timestamp","YYYY-MM-dd-HH.mm.ss.SSSSSS"]
}
}
output {
stdout {
codec=>rubydebug
}
}
3.pattern.txt
DB2TIME %{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}-%{HOUR:hour}.%{MINUTE:minute}.%{SECOND:second}