flume 拓扑实战

一、需求

1. flume-1监控test.txt日志，flume-1的数据传送给flume-2，flume-2将数据追加到本地文件，同时flume-2将数据传输到flume-3。
2. flume-4监控本地另一个自己创建的文件any.txt，并将数据传送给flume-3。
3. flume-3将汇总数据写入到HDFS。

二、分析

2.1 拓扑结构

2.2 配置文件

flume1.conf

a1.sources = s1
a1.sinks = k1
a1.channels = c1

a1.sources.s1.type = TAILDIR
a1.sources.s1.positionFile = /tmp/flume/data/taildir_position1.json
a1.sources.s1.filegroups = f1
a1.sources.s1.filegroups.f1 = /tmp/flume/data/test.txt

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sinks.k1.type = avro
a1.sinks.k1.hostname = master
a1.sinks.k1.port = 11000

a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

flume2.conf

a2.sources = s1
a2.sinks = k1 k2
a2.channels = c1 c2

a2.sources.s1.type = avro
a2.sources.s1.bind = master
a2.sources.s1.port = 11000

a2.sources.s1.selector.type = replicating

a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100

a2.channels.c2.type = memory
a2.channels.c2.capacity = 1000
a2.channels.c2.transactionCapacity = 100

a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /tmp/flume/data/
a2.sinks.k1.sink.rollInterval = 3600

a2.sinks.k2.type = avro
a2.sinks.k2.hostname = master
a2.sinks.k2.port = 12000

a2.sources.s1.channels = c1 c2
a2.sinks.k1.channel = c1
a2.sinks.k2.channel = c2

flume3.conf

a3.sources = s1 s2
a3.sinks = k1
a3.channels = c1

a3.sources.s1.type = avro
a3.sources.s1.bind = master
a3.sources.s1.port = 12000

a3.sources.s2.type = avro
a3.sources.s2.bind = master
a3.sources.s2.port = 13000

a3.channels.c1.type = memory
a3.channels.c1.capacity = 2000
a3.channels.c1.transactionCapacity = 200

a3.sinks.k1.type = hdfs
a3.sinks.k1.hdfs.path = /flume/%Y-%m-%d/%H
a3.sinks.k1.hdfs.filePrefix = file-
a3.sinks.k1.hdfs.fileType = DataStream
a3.sinks.k1.hdfs.round = true
a3.sinks.k1.hdfs.roundValue = 1
a3.sinks.k1.hdfs.roundUnit = hour
a3.sinks.k1.hdfs.rollInterval = 0
a3.sinks.k1.hdfs.rollSize = 134217000
a3.sinks.k1.hdfs.rollCount = 0
a3.sinks.k1.hdfs.useLocalTimeStamp = true


a3.sources.s1.channels = c1
a3.sources.s2.channels = c1
a3.sinks.k1.channel = c1

flume4.conf

a4.sources = s1
a4.sinks = k1
a4.channels = c1

a4.sources.s1.type = TAILDIR
a4.sources.s1.positionFile = /tmp/flume/data/taildir_position2.json
a4.sources.s1.filegroups = f1
a4.sources.s1.filegroups.f1 = /tmp/flume/data/any.txt

a4.channels.c1.type = memory
a4.channels.c1.capacity = 1000
a4.channels.c1.transactionCapacity = 100

a4.sinks.k1.type = avro
a4.sinks.k1.hostname = master
a4.sinks.k1.port = 13000

a4.sources.s1.channels = c1
a4.sinks.k1.channel = c1