flume_kafkaChannel_kafkaSink

2019独角兽企业重金招聘Python工程师标准>>>

agent.sources = source
#抽取类型为目录
agent.sources.source.type = spooldir
#抽取的文件目录
agent.sources.source.spoolDir = /root/tmp/flume/data
#添加一个存储绝对路径文件名的头
agent.sources.source.fileHeader = true
#never or immediate 收集文件完成之后是否删除
agent.sources.source.deletePolicy = immediate
#监视子目录以查找新文件。
agent.sources.source.recursiveDirectorySearch = false
#批量传输到通道的粒度
agent.sources.source.batchSize = 20
#解串器使用的字符集，将输入文件视为文本
agent.sources.source.inputCharset = UTF-8
#输入文件中出现不可解码的字符FAIL，REPLACE，IGNORE
agent.sources.source.decodeErrorPolicy = REPLACE

agent.sources.source.channels = channel
#kafka  通道
agent.channels  =  channel
agent.channels.channel.type = org.apache.flume.channel.kafka.KafkaChannel
agent.channels.channel.kafka.bootstrap.servers = slave1:9092
agent.channels.channel.kafka.topic = channeltopic
agent.channels.channel.kafka.consumer.group.id = flume-consumer
#偏移重置earliest,latest,none
agent.channels.channel.kafka.consumer.auto.offset.reset = none
agent.channels.channel.kafka.pollTimeout = 1000

#file  通道
#agent.channels  =  channel
#agent.channels.channel.type  =  file 
#agent.channels.channel.checkpointDir  =  /root/tmp/flume/checkpoint 
#agent.channels.channel.checkpointOnClose = true
#agent.channels.channel.dataDirs  =  /root/tmp/flume/channeldata

agent.sinks.sink.channel = channel

agent.sinks = sink
agent.sinks.sink.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.sink.kafka.topic = test
agent.sinks.sink.kafka.bootstrap.servers = slave1:9092
#一个批处理中处理的消息数
agent.sinks.sink.kafka.flumeBatchSize = 100
#0（从不等待确认），1（仅等待领导者），-1（等待所有副本）
agent.sinks.sink.kafka.producer.acks = -1
agent.sinks.sink.kafka.producer.linger.ms = 1
agent.sinks.sink.kafka.producer.compression.type = snappy
 

转载于:https://my.oschina.net/u/2510243/blog/1796761