[日志处理工作之五]整合logstash,kafka

1.logstash filter component组件的正则表达式要写的足够strong，需要充分考虑字段值为空的情况，比如DB字段，如果不考虑，数据会丢失。
logstash内置了kafka output组件。 file input基本实现了tail的功能
logstash-kafka.conf
input{
  exec{
    command => "cat /db2fs/opt/log-process/share/db2_test.log"
    codec => multiline{
      pattern => "?\d\d\d\d-\d\d"
      negate => true
      what => previous
    }
    interval=>30
  }
  #file{
   # path => "/db2fs/opt/log-process/share/db2_test.log"
   # codec => multiline{
   #   pattern => "?\d\d\d\d-\d\d"
   #   negate => true
   #   what => previous
   # }
  #}
}


filter {
  mutate{
    gsub => ['message', "\n", " "]
  }
}


output {
##  elasticsearch { host => localhost }
  kafka {
    topic_id=>"stream"
    broker_list=>"9.115.42.108:9092"
  }
}


2.kafka-elastic .conf
input{
  kafka{
    topic_id => "stream"
  }
}


filter {
  grok{
    patterns_dir=>"/db2fs/opt/log-process/logstash-1.5.0/conf/patterns"
    match=>{"message"=>"%{DB2TIME:timestamp}%{INT:timezone}\s*%{WORD:recordid}\s*LEVEL\s*:%{DATA:level}PID\s*:\s*%{NUMBER:processid}\s*TID\s*:\s*%{NUMBER:threadid}\s*PROC\s*:%{DATA:process}INSTANCE\s*:\s*%{WORD:instance}\s*NODE\s*:\s*%{WORD:node}\s*(?:DB\s*:\s%{DBNAME:dbname}\s*)?(?:APPHDL\s*:\s*%{NOTSPACE:apphdl}\s*)?(?:APPID\s*:\s*%{NOTSPACE:appid}\s*)?(?:AUTHID\s*:\s*%{WORD:authid}\s*)?(?:HOSTNAME\s*:\s*%{NOTSPACE:hostname}\s*)?(?:EDUID\s*:\s*%{NUMBER:eduid}\s*)?(?:EDUNAME\s*:\s*%{DATA:eduname}\s*)?FUNCTION\s*:%{DATA:function}probe\s*:\s*%{NUMBER:probe}\s*%{GREEDYDATA:functionlog}"}
  }
  date{
    match=>["timestamp","YYYY-MM-dd-HH.mm.ss.SSSSSS"]
  }
}
output {
  stdout {
    codec=>rubydebug
  }
}


3.pattern.txt
DB2TIME  %{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}-%{HOUR:hour}.%{MINUTE:minute}.%{SECOND:second}