flume常见agent,source,sink,channel,复制和多路复用，故障转移，聚合的使用

source

• netcat

  a1.sources.r1.type=netcat
  a1.sources.r1.bind=localhost
  a1.sources.r1.port=26001
  

• taildir

  a1.sources.r1.type=taildir
  a1.sources.r1.filegroups=f1
  a1.sources.r1.filefroups.f1=/user/flume
  

• exec

  a1.sources.r1.type=exec
  a1.sources.r1.command=tail -F /usr/flume
  
  

• spooldir

  a1.sources.r1.type=spooldir
  a1.sources.r1.spoolDir=/usr/flume
  

• avro

  a1.sources.r1.type=avro
  a1.sources.r1.bind=0.0.0.0
  a1.sources.r1.port=4141
  

• kafka source

  a1.sources.r1.type=org.apache.flume.source.kafka.KafkaSource
  a1.sources.r1.kafka.bootstrap.servers=
  a1.sources.r1.kafka.topics=
  a1.sources.r1.kafka.topics.regex=
  

Sink

• kafka

  a1.sinks.k1.tyep=org.apache.flume.sink.kakfa.KafkaSink
  a1.sinks.k1.kafka.topic=order
  a1.sinks.k1.kafka.bootstrap.servers=master:9092
  

• hdfs

  a1.sinks.k1.type=hdfs
  a1.sinks.k1.hdfs.path=/user/flumebackup
  a1.sinks.k1.hdfs.fileType=DataStream
  a1.sinks.k1.hdfs.rollSize=0
  a1.sinks.k1.hdfs.rollInterval=0
  a1.sinks.k1.hdfs.rollCount=0
  

• logger

  a1.sinks.k1.type=logger
  

• avro

  a1.sinks.k1.type=avro
  a1.sinks.k1.hostname=master
  a1.sinks.k1.port=4545
  

channel

• memory

  a1.channels.c1.type=memory
  a1.channels.c1.capacity=1000
  a1.channels.c1.transactionCapacity=100
  

• file

  a1.channels.c1.type=file
  a1.channels.c1.checkpointDir=
  

• kafka

  a1.channels.c1.type=org.apache.flume.channel.kafka.KafkaChannel
  a1.channels.c1.kafka.bootstrap.servers=master:9092
  a1.channels.c1.topic=order
  

简单串联

复制和多路复用

a1.sources=r1
a1.sinks=k1 k2
a1.channels=c1 c2

a1.sources.r1.type=netcat
a1.sources.r1.bind=localhost
a1.sources.r1.port=2999

a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=/user/test
a1.sinks.k1.hdfs.fileType=DataStream

a1.sinks.k2.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k2.kafka.topic=order
a1.sinks.k2.kafka.bootstrap.servers=master:9092

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.channels.c2.type=memory
a1.channels.c2.capacity=1000
a1.channels.c2.transactionCapacity=100

a1.sources.r1.channels=c1 c2
a1.sinks.k1.channel=c1
a1.sinks.k2.channel=c2

负载均衡和故障转移

a1.sources=r1
a1.sinks=k1 k2
a1.channels=c1

a1.sources.r1.type=exec
a1.sources.r1.command=tail -F /usr/test/test.txt

a1.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers=master:9092
a1.sinks.k1.kafka.topic=order

a1.sinks.k2.type=logger

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
a1.sinks.k2.channel=c1

聚合

a1.sources=r1
a1.sinks=k1
a1.channels=c1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/usr/dbtest/test.txt

a1.sinks.k1.type=avro
a1.sinks.k1.hostname=slave2
a1.sinks.k1.port=555

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1