flume1.7.0 常用配置

一、source 为spooling dir

a1.sources = r1
a1.channels = c1
a1.sinks = k1

a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /opt/nginx/test
a1.sources.r1.channels = c1
a1.sources.r1.fileSuffix = .pro
a1.sources.r1.includePattern = ^.*$

a1.channels.c1.type = memory
a1.channels.c1.capacity = 6000
a1.channels.c1.transactionCapacity = 100

a1.sinks.k1.type = com.sohu.tv.flume.sink.TestSink
a1.sinks.k1.channel = c1
a1.sinks.k1.kafka.topic = test
a1.sinks.k1.kafka.batchsize = 1
a1.sinks.k1.serializer.class = kafka.serializer.StringEncoder
a1.sinks.k1.key.serializer = org.apache.kafka.common.serialization.StringSerializer
a1.sinks.k1.value.serializer = org.apache.kafka.common.serialization.StringSerializer
a1.sinks.k1.compression.codec = 2
a1.sinks.k1.bootstrap.servers=10.10.*.201:9092
#a1.sinks.k1.metadata.broker.list=10.10.*.194:9092,10.10.*.204:9092
a1.sinks.k1.request.required.acks=1
a1.sinks.k1.producer.type=async
a1.sinks.k1.queue.buffering.max.messages=1
a1.sinks.k1.batch.num.messages=1

二、sink为hdfs

a1.sources = r1
a1.channels = c2
a1.sinks = k2

a1.sources.r1.type = exec
a1.sources.r1.command = tail -n +0 -F /opt/nginx/logs/link_pt.log
a1.sources.r1.channels = c2
a1.sources.r1.batchSize = 200
#a1.sources.r1.useHost = true

#batch代表一次向通道提交的量，或者一次从通道取的量
#a1.channels.c2.type = memory
#a1.channels.c2.capacity = 10000   通道的容量
#a1.channels.c2.keep-alive = 6
#a1.channels.c2.transactionCapacity = 1000  事务提交容量，与batchsize大小差不多，大于等于batch

a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c2.capacity = 10000
a1.channels.c2.brokerList = 10.10.*.162:9092,10.10.*.104:9092,10.10.*.108:9092
a1.channels.c2.topic = adlog_channel
a1.channels.c2.groupId = adlog88161
a1.channels.c2.zookeeperConnect = 10.10.*.194:2181,10.10.*.204:2181,10.10.*.125:2181
a1.channels.c2.transactionCapacity = 1000

a1.sinks.k2.type = hdfs
a1.sinks.k2.channel = c2
a1.sinks.k2.hdfs.path = hdfs://buffercluster1/user/hive/warehouse/adlog_text/%Y%m%d/%Y%m%d%H
a1.sinks.k2.hdfs.useLocalTimeStamp = true
a1.sinks.k2.hdfs.filePrefix = adlogaccess_-%Y%m%d%H
#a1.sinks.k2.hdfs.filePrefix = pvaccess-
a1.sinks.k2.hdfs.fileType = DataStream
a1.sinks.k2.hdfs.writeFormat = Text
a1.sinks.k2.hdfs.round = true
a1.sinks.k2.hdfs.roundValue = 1
a1.sinks.k2.hdfs.roundUnit = hour
a1.sinks.k2.hdfs.callTimeout = 30000
a1.sinks.k2.hdfs.rollInterval  = 3600
a1.sinks.k2.hdfs.rollSize = 0
a1.sinks.k2.hdfs.rollCount  = 0

三、sink为kafka

a1.sources = r1
a1.channels = c1
a1.sinks = k1

a1.sources.r1.type = exec
a1.sources.r1.command = tail -n +0 -F /opt/log/link_vv.log
a1.sources.r1.channels = c1

a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 1000

a1.sinks.k1.type = KafkaSink
a1.sinks.k1.channel = c1
a1.sinks.k1.kafka.topic = topic1,topic2
a1.sinks.k1.kafka.batchsize = 300
a1.sinks.k1.serializer.class = kafka.serializer.StringEncoder
a1.sinks.k1.compression.codec = 2
a1.sinks.k1.metadata.broker.list=10.10.*.51:9092,10.10.*.52:9092,10.10.*.53:9092
a1.sinks.k1.request.required.acks=1
a1.sinks.k1.producer.type=async
a1.sinks.k1.queue.buffering.max.messages=2000
a1.sinks.k1.batch.num.messages=1000