elk 实例

input{
 kafka {
      zk_connect => "fuze245:2181,fuze246:2181"
      group_id => "nginx_es"
      topic_id => "nginx_es"
      reset_beginning => false # boolean (optional)， default: false
      consumer_threads => 2  # number (optional)， default: 1
      decorate_events => true # boolean (optional)， default: false
      codec => plain { 
        format => "%{message}"
         }
  } 
}

filter {
    grok {
        match => {
        #%{IP:ip} \[%{DATA:timestamp} %{WORD:level} \] %{JAVACLASS}:(?<params>[\s\S]*)
                patterns_dir => "./patterns"
           "message" => "%{IP:client_ip} - %{USER:client} \[%{DATA:datestamp}\] \"%{WORD:request_type} %{URIPATHPARAM:request_methods} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} (?:%{NUM
BER:bytes}|-) (?:\"(?:%{URI:referer}|-)\"|%{QS:referer}) %{QS:agent} %{USER:xforwardedfor} \"%{URIHOST:upstream_host}\" \"%{NUMBER:upstream_status}\" \"%{NUMBER:response_time}\" \"%{NUMBER:
request_time}\""
        }
       # date {
       # match => ["logdate", "dd/MMM/yyyy:HH:mm:ss Z"]
         # }
    }
}

output{
#    stdout {
#       codec => rubydebug
#     }
    if "_grokparsefailure" not in [tags] {
     elasticsearch {
            hosts => "192.168.1.246:9200"
            index => "nginx"
      }
    }else{
        elasticsearch {
                hosts => "192.168.1.246:9200"
                index => "error"
          }
    }
 }