多个Flume合并一个channel上传文件到Hdfs

需要交流请进群-- 494831900 --我和群中朋友会及时回答

需要交流请进群-- 494831900

在tohdfs.conf 中配置如下

--------------------------------------------------------------------------------------------------
agent1表示代理名称
agent1.sources=source1 #只能也仅有一个源 
agent1.sinks=sink1 sink2  #多个sinks
agent1.channels=channel1 channel2  #多个channel
#配置channel1
agent1.channels.channel1.type=file
agent1.channels.channel1.checkpointDir=/usr/wang/flume
agent1.channels.channel1.dataDirs=/usr/wang/flume/data
#配置channel2
agent1.channels.channel2.type = memory
agent1.channels.channel2.capacity = 1000000
agent1.channels.channel2.transactionCapacity = 100000
agent1.channels.channel2.keep-alive = 10 
#配置source1
#agent1.sources.source1.type=spooldir
#agent1.sources.source1.spoolDir=/usr/local/tomcat/wanglog
#agent1.sources.source1.channels=channel1 channel2
#agent1.sources.source1.batchSize=100
#agent1.sources.source1.ignorePattern=log.log
#agent1.sources.source1.deletePolicy=immediate

agent1.sources.source1.channels =channel1 channel2
agent1.sources.source1.type = avro
agent1.sources.source1.bind = master
agent1.sources.source1.port = 44444
agent1.sources.source1.threads = 5
#实时的获取日志文件
#agent1.sources.source1.type=exec
#agent1.sources.source1.command =tail -n +0 -F /usr/local/tomcat/wanglog/log.log
#agent1.sources.source1.channels=channel1
#配置sink1
agent1.sinks.sink1.type=hdfs
agent1.sinks.sink1.hdfs.path=hdfs://master:9000/tomcat
agent1.sinks.sink1.hdfs.fileType=DataStream
agent1.sinks.sink1.hdfs.writeFormat=Text
agent1.sinks.sink1.hdfs.rollInterval=4
agent1.sinks.sink1.hdfs.filePrefix=wang
agent1.sinks.sink1.hdfs.fileSuffix=test
agent1.sinks.sink1.channel=channel1
agent1.sinks.sink1.hdfs.rollCount=50
agent1.sinks.sink1.hdfs.rollSize=2048
#配置sink2
agent1.sinks.sink2.type = file_roll
agent1.sinks.sink2.channel = channel2
agent1.sinks.sink2.sink.directory =/usr/local/testdir

在totomcat.conf 中配置如下
--------------------------------------------------------------------------------------------------
agent1表示代理名称
agent1.sources=source1
agent1.sinks=sink1
agent1.channels=channel2

#配置source1
agent1.sources.source1.type=spooldir
agent1.sources.source1.spoolDir=/usr/local/tomcat/wanglog
agent1.sources.source1.channels=channel2
agent1.sources.source1.batchSize=100
agent1.sources.source1.ignorePattern=log.log
agent1.sources.source1.deletePolicy=immediate

#实时的获取日志文件
#agent1.sources.source1.type=exec
#agent1.sources.source1.command =tail -n +0 -F /usr/local/tomcat/wanglog/log.log
#agent1.sources.source1.channels=channel2

#配置上传到hdfs的sink1
#agent1.sinks.sink1.channel=channel2
#agent1.sinks.sink1.type=hdfs
#agent1.sinks.sink1.hdfs.path=hdfs://master:9000/tomcat
#agent1.sinks.sink1.hdfs.fileType=SequenceFile
#agent1.sinks.sink1.hdfs.writeFormat=Text
#agent1.sinks.sink1.hdfs.rollInterval=4
#agent1.sinks.sink1.hdfs.filePrefix=wang
#agent1.sinks.sink1.hdfs.fileSuffix=test
#agent1.sinks.sink1.hdfs.rollCount=50
#agent1.sinks.sink1.hdfs.rollSize=2048

#配置上传到另外一个channal 的flume 的sink1
agent1.sinks.sink1.type = avro
agent1.sinks.sink1.hostname=master
agent1.sinks.sink1.port = 44444
agent1.sinks.sink1.channel = channel2

#配置channel2
agent1.channels.channel2.type=memory
agent1.channels.channel2.capacity = 10000
agent1.channels.channel2.transactionCapacity = 10000
agent1.channels.channel2.keep-alive = 3