kafka的数据来源于Nginx的日志。
#源数据
192.168.1.123 - - [15/May/2020:21:47:39 +0800] "GET /nas/ma/q.gif?a=123&b=message&p=12345678901&b=p&timer=1589550459586 HTTP/1.1" 192.168.32.118:80 0.002 200
采集后的到kafka的数据
{
"@timestamp":"2020-05-15T13:47:43.216Z",
"@metadata":{
"beat":"filebeat",
"type":"_doc",
"version":"7.2.0",
"topic":"bigdata_nas_access"
},
"message":"192.168.1.123 - - [15/May/2020:21:47:39 +0800] "GET /nas/ma/q.gif?a=123&b=message&p=12345678901&b=p&timer=1589550459586 HTTP/1.1" 192.168.32.118:80 0.002 200"
}
logstash 脚本
input{
kafka {
bootstrap_servers => ["192.168.1.68:9092,192.168.1.69:9092,192.168.1.67:9092"]
client_id => "bigdata_88"
group_id => "bigdata_nas_access_88"
auto_offset_reset => "latest"
consumer_threads => 3
decorate_events => true
topics => ["bigdata_nas_access"]
type => "nas_access"
codec => 'json'
}
}
filter {
if [type] == "nas_access" {
grok {
match => {
"message" => "%{IPORHOST:Client_IP} (%{WORD:ident}|-) (%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:Http_Method} %{URIPATHPARAM:Http_Request} HTTP/%{NUMBER:Http_Version}\" (?:%{HOSTPORT:upstream_addr}|-) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE10NUM:upstream_status}|-)"
}
}
#替换时间
date {
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
}
#切割request
mutate{
split=>["request","?"]
add_field => {
"url" => "%{[request][0]}"
}
add_field => {
"param" => "%{[request][1]}"
}
remove_field => ["request"]
}
#KV参数
kv {
source => "param"
field_split => "&"
target => "kv"
}
mutate{ remove_field => ["param","tags","timestamp"] }
}
}
output {
stdout { codec => rubydebug }
if [type] == "nas_access" {
elasticsearch {
hosts => ["192.168.1.86:9200","192.168.1.87:9200","192.168.1.88:9200"]
index => "nas_access"
document_id => "%{Client_IP}" #用来判断唯一性
}
}