source
-
netcat
a1.sources.r1.type=netcat a1.sources.r1.bind=localhost a1.sources.r1.port=26001
-
taildir
a1.sources.r1.type=taildir a1.sources.r1.filegroups=f1 a1.sources.r1.filefroups.f1=/user/flume
-
exec
a1.sources.r1.type=exec a1.sources.r1.command=tail -F /usr/flume
-
spooldir
a1.sources.r1.type=spooldir a1.sources.r1.spoolDir=/usr/flume
-
avro
a1.sources.r1.type=avro a1.sources.r1.bind=0.0.0.0 a1.sources.r1.port=4141
-
kafka source
a1.sources.r1.type=org.apache.flume.source.kafka.KafkaSource a1.sources.r1.kafka.bootstrap.servers= a1.sources.r1.kafka.topics= a1.sources.r1.kafka.topics.regex=
Sink
-
kafka
a1.sinks.k1.tyep=org.apache.flume.sink.kakfa.KafkaSink a1.sinks.k1.kafka.topic=order a1.sinks.k1.kafka.bootstrap.servers=master:9092
-
hdfs
a1.sinks.k1.type=hdfs a1.sinks.k1.hdfs.path=/user/flumebackup a1.sinks.k1.hdfs.fileType=DataStream a1.sinks.k1.hdfs.rollSize=0 a1.sinks.k1.hdfs.rollInterval=0 a1.sinks.k1.hdfs.rollCount=0
-
logger
a1.sinks.k1.type=logger
-
avro
a1.sinks.k1.type=avro a1.sinks.k1.hostname=master a1.sinks.k1.port=4545
channel
-
memory
a1.channels.c1.type=memory a1.channels.c1.capacity=1000 a1.channels.c1.transactionCapacity=100
-
file
a1.channels.c1.type=file a1.channels.c1.checkpointDir=
-
kafka
a1.channels.c1.type=org.apache.flume.channel.kafka.KafkaChannel a1.channels.c1.kafka.bootstrap.servers=master:9092 a1.channels.c1.topic=order
简单串联
复制和多路复用
a1.sources=r1
a1.sinks=k1 k2
a1.channels=c1 c2
a1.sources.r1.type=netcat
a1.sources.r1.bind=localhost
a1.sources.r1.port=2999
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=/user/test
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k2.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k2.kafka.topic=order
a1.sinks.k2.kafka.bootstrap.servers=master:9092
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.channels.c2.type=memory
a1.channels.c2.capacity=1000
a1.channels.c2.transactionCapacity=100
a1.sources.r1.channels=c1 c2
a1.sinks.k1.channel=c1
a1.sinks.k2.channel=c2
负载均衡和故障转移
a1.sources=r1
a1.sinks=k1 k2
a1.channels=c1
a1.sources.r1.type=exec
a1.sources.r1.command=tail -F /usr/test/test.txt
a1.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers=master:9092
a1.sinks.k1.kafka.topic=order
a1.sinks.k2.type=logger
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
a1.sinks.k2.channel=c1
聚合
a1.sources=r1
a1.sinks=k1
a1.channels=c1
a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/usr/dbtest/test.txt
a1.sinks.k1.type=avro
a1.sinks.k1.hostname=slave2
a1.sinks.k1.port=555
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1