flume 单个Agent 多对一 多个sources 单个channels 单个sinks

运行

flume-ng agent \
--conf $FLUME_HOME/conf/ \
--name many2one \
--conf-file /opt/test/homework/flume.conf \
-Dflume.root.logger==INFO,console

conf配置文件

# agent=>many2one
many2one.sources=source1 source2
many2one.sinks=sink1
many2one.channels=c1

# 2.source1对应c1 传入bigdata111:4666端口信息
many2one.sources.source1.type = netcat
many2one.sources.source1.bind = bigdata111
many2one.sources.source1.port = 4666

# 3.source2对应c2 传入Hadoop的日志文件信息
many2one.sources.source2.type = exec
many2one.sources.source2.command = tail -F /opt/module/hadoop-3.1.2/logs/hadoop-root-namenode-bigdata111.log
many2one.sources.source2.shell = /bin/bash -c

# 4.定义sink
many2one.sinks.sink1.type = hdfs
many2one.sinks.sink1.hdfs.path = hdfs://bigdata111:9000/bigdataHomeWork/%H%M
#上传文件的前缀
many2one.sinks.sink1.hdfs.filePrefix = logs-
#是否按照时间滚动文件夹
many2one.sinks.sink1.hdfs.round = true
#多少时间单位创建一个新的文件夹
many2one.sinks.sink1.hdfs.roundValue = 1
#重新定义时间单位
many2one.sinks.sink1.hdfs.roundUnit = hour
#是否使用本地时间戳
many2one.sinks.sink1.hdfs.useLocalTimeStamp = true
#积攒多少个Event才flush到HDFS一次
many2one.sinks.sink1.hdfs.batchSize = 100
#设置文件类型，可支持压缩
many2one.sinks.sink1.hdfs.fileType = DataStream
#多久生成一个新的文件
many2one.sinks.sink1.hdfs.rollInterval = 600
#设置每个文件的滚动大小
many2one.sinks.sink1.hdfs.rollSize = 134217728
#文件的滚动与Event数量无关
many2one.sinks.sink1.hdfs.rollCount = 0
#最小副本数
many2one.sinks.sink1.hdfs.minBlockReplicas = 1

# 5.定义channel
many2one.channels.c1.type = memory
many2one.channels.c1.capacity = 1000
many2one.channels.c1.transactionCapacity = 100

# 6.连接
many2one.sources.source1.channels = c1
many2one.sources.source2.channels = c1
many2one.sinks.sink1.channel = c1