Flume TAILDIR 生产实践

# bin/flume-ng agent -c conf -f conf/hdfs-kafka-sink.conf -n a1
# nohup /server/app/apache-flume-1.8.0-bin/flume-ng agent  -c /server/app/apache-flume-1.8.0-bin/conf -f /server/app/apache-flume-1.8.0-bin、conf/flume-taildir-conf.properties -n a1  -Dflume.root.logger=INFO,console >> daFlume.log  &
# Name the components on this agent
a1.sources = r1
#a1.sinks = k1 k2
a1.sinks = k1
a1.channels = c1




# Describe/configure the source


# TailDIR
a1.sources.r1.type = TAILDIR
a1.sources.r1.positionFile = /home/sznongfu/tmp/taildir_hdfs_kafka.json
#a1.sources.r1.skipToEnd = true
#a1.sources.r1.batchSize = 2000
a1.sources.r1.batchSize = 1000
a1.sources.r1.writePosInterval= 1000
a1.sources.r1.idleTimeout = 6000


# one filegroup
#a1.sources.r1.filegroups = f1 
#a1.sources.r1.filegroups.f1 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/.*api\.log
#a1.sources.r1.filegroups.f1 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/.*api.log
###自定义保存到header中的key value
##a1.sources.r1.headers.f1.fileName = access.log.1
#a1.sources.r1.fileHeader = true
#a1.sources.r1.fileHeaderKey = file




#five file group 


a1.sources.r1.filegroups = f1 f2 f3 f4 f5




a1.sources.r1.fileHeader = true
a1.sources.r1.headers.f1.headerKey1 = value1
a1.sources.r1.headers.f2.headerKey2 = value2
a1.sources.r1.headers.f3.headerKey3 = value3
a1.sources.r1.headers.f4.headerKey4 = value4
a1.sources.r1.headers.f5.headerKey5 = value5






a1.sources.r1.filegroups.f1 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/nongfu.merchant.client.api.log
a1.sources.r1.filegroups.f2 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/nongfu.merchant.cmn.api.log
a1.sources.r1.filegroups.f3 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/nongfu.merchant.dashboard.api.log
a1.sources.r1.filegroups.f4 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/nongfu.merchant.passport.api.log
a1.sources.r1.filegroups.f5 = /home/sznongfu/opt/nginx-1.9/logs/merchant-master-logs/nongfu.merchant.repo.api.log






# Describe the sink
#a1.sinks.k1.type = hdfs
#a1.sinks.k1.channel = c1
## 每个一个小时产生一个日志目录
#a1.sinks.k1.hdfs.path = hdfs://hadoop:19000/dev/nginx/log/nongfu.mw/raw/%Y-%m-%d
#a1.sinks.k1.hdfs.filePrefix = trace_%H
#a1.sinks.k1.hdfs.fileSuffix = .log
#a1.sinks.k1.hdfs.round = false
#a1.sinks.k1.hdfs.roundValue = 1
#a1.sinks.k1.hdfs.roundUnit = hour
#a1.sinks.k1.hdfs.rollInterval = 0
#a1.sinks.k1.hdfs.rollSize = 134217728
#a1.sinks.k1.hdfs.rollCount = 0
#a1.sinks.k1.hdfs.idleTimeout = 60
#a1.sinks.k1.hdfs.batchSize = 100
#a1.sinks.k1.hdfs.minBlockReplicas=1  
#a1.sinks.k1.hdfs.useLocalTimeStamp = true
##生成的文件类型，默认是Sequencefile，可用DataStream，则为普通文本
#a1.sinks.k1.hdfs.fileType = DataStream




 # kafka sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.channel = c1
a1.sinks.k1.kafka.topic = nongfunginxlog
a1.sinks.k1.kafka.bootstrap.servers = hadoop:19092,spark:19092
#a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.flumeBatchSize = 200
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
a1.sinks.k1.kafka.producer.compression.type = snappy










 #file sink
#a1.sinks.k2.type = file_roll
#a1.sinks.k2.channel = c1
#a1.sinks.k2.sink.directory = /tmp/flumelog






# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 100000
a1.channels.c1.transactionCapacity = 100000






# file channel
#a1.channels.c1.type = file
#a1.channels.c1.checkpointDir = /tmp/flumechannel/checkpoint
#a1.channels.c1.dataDirs = /tmp/flumechannel/data






# file channel
#a1.channels.c1.type = file
#a1.channels.c1.checkpointDir = /tmp/flumechannel/checkpoint
#a1.channels.c1.dataDirs = /tmp/flumechannel/data








# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channels = c1
#a1.sinks.k2.channels = c1