logstash消费kafka的数据

kafka的数据来源于Nginx的日志。

#源数据
192.168.1.123 - - [15/May/2020:21:47:39 +0800]  "GET /nas/ma/q.gif?a=123&b=message&p=12345678901&b=p&timer=1589550459586 HTTP/1.1" 192.168.32.118:80 0.002 200

采集后的到kafka的数据

{
    "@timestamp":"2020-05-15T13:47:43.216Z",
    "@metadata":{
        "beat":"filebeat",
        "type":"_doc",
        "version":"7.2.0",
        "topic":"bigdata_nas_access"
    },
    "message":"192.168.1.123 - - [15/May/2020:21:47:39 +0800]  "GET /nas/ma/q.gif?a=123&b=message&p=12345678901&b=p&timer=1589550459586 HTTP/1.1" 192.168.32.118:80 0.002 200"
}

logstash 脚本

input{
kafka {
    bootstrap_servers => ["192.168.1.68:9092,192.168.1.69:9092,192.168.1.67:9092"]
    client_id => "bigdata_88"
    group_id => "bigdata_nas_access_88"
    auto_offset_reset => "latest"
    consumer_threads => 3
    decorate_events => true
    topics => ["bigdata_nas_access"]
    type => "nas_access"
    codec => 'json'
  }
}
filter {
  if [type] == "nas_access" {
        grok {
            match => {
                "message" => "%{IPORHOST:Client_IP} (%{WORD:ident}|-) (%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\]  \"%{WORD:Http_Method} %{URIPATHPARAM:Http_Request} HTTP/%{NUMBER:Http_Version}\" (?:%{HOSTPORT:upstream_addr}|-) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE10NUM:upstream_status}|-)"
            }
        }
        #替换时间
        date {
           match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
           target => "@timestamp"
        }
        #切割request
        mutate{
           split=>["request","?"]
                add_field => {
                        "url" => "%{[request][0]}"
                }
                add_field => {
                        "param" => "%{[request][1]}"
                }
                remove_field => ["request"]
        }
        #KV参数
        kv {
                source => "param"
                field_split => "&"
                target => "kv"
        }
        mutate{ remove_field => ["param","tags","timestamp"] }
  }
}
output {
        stdout { codec => rubydebug }
 if [type] == "nas_access" {
     elasticsearch {
     hosts => ["192.168.1.86:9200","192.168.1.87:9200","192.168.1.88:9200"]
     index => "nas_access"
     document_id => "%{Client_IP}"  #用来判断唯一性
   }
}