input{
kafka{
bootstrap_servers => ["192.169.160.25:7091,192.169.160.26:7092,192.169.160.27:7093,192.169.160.28:7094,192.169.160.29:7095"] #kafka的地址,集群有逗号隔开
client_id => "cpm_addUser"
group_id => "cpm_addUser"
auto_offset_reset => "latest" #从最新的偏移量开始消费
consumer_threads => 5
decorate_events => true #此属性会将当前topic、offset、group、partition等信息也带到message中
topics => ["cpm_addUser"]
type => "cpm_adduser"
}
}
filter {
if ([message]!= "")
{
if([type]=="cpm_adduser"){
mutate{
split => ["message","||"]
add_field => {
"username" => "%{[message][0]}"
}
add_field => {
"cpm_strategy" => "%{[message][1]}"
}
add_field => {
"time" => "%{[message][2]}"
}
#删除message字段
remove_field =>["message"]
}}
} else{
drop {
}
}
#将时间弄成东八区
ruby {
code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)"
}
ruby{
code => "event.set('day', (event.get('@timestamp').time.localtime + 8*60*60).strftime('%Y.%m.%d'))"
}
mutate{
#转换数据类型为integer
convert => ["time","integer"]
}
}
output {
if [type] == "cpm_adduser"{
elasticsearch {
hosts => ["192.169.160.18:9200"] # ElasticSearch的地址加端口
index => "cpm_adduser" # ElasticSearch的保存文档的index名称,
}
}
}
WARN [Consumer clientId=consumer-1, groupId=console-consumer-950] Connection to node -1 could not be
https://blog.csdn.net/getyouwant/article/details/79000524
kafka安装及快速入门
https://blog.csdn.net/yjt520557/article/details/88558065
更多信息:https://www.cnblogs.com/willpan-z/p/10307967.html?tdsourcetag=s_pctim_aiomsg
点赞或者评论是我最大的动力,有问题欢迎留言或者联系q:1559810637