flume架构图以及模板

模板：

================主要方式===================

a1.sources = r1  
a1.channels = c1 c2
a1.sinks = s1 s2

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir =/root/my
a1.sources.r1.selector.type=multiplexing        //指定multiplexing selector 会比较header信息，不指定会有默认的
a1.sources.r1.selector.header = state
a1.sources.r1.selector.mapping.USER = c1
a1.sources.r1.selector.mapping.ORDER = c2
a1.sources.r1.selector.default = c1
a1.sources.r1.interceptors=i1 i2 i3
a1.sources.r1.interceptors.i1.type=timestamp
a1.sources.r1.interceptors.i1.preserveExisting=false
a1.sources.r1.interceptors.i2.type=host
a1.sources.r1.interceptors.i2.preserveExisting=false
a1.sources.r1.interceptors.i2.useIP=true
a1.sources.r1.interceptors.i2.hostHeader=hostname
a1.sources.r1.interceptors.i3.type=static
a1.sources.r1.interceptors.i3.preserveExisting=false
a1.sources.r1.interceptors.i3.key=hn
a1.sources.r1.interceptors.i3.value=hdp

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000                    
a1.channels.c1.transactionCapacity=100           
a1.channels.c1.keep-alive=3
a1.channels.c1.byteCapacityBufferPercentage=20    
a1.channels.c1.byteCapacity=800000               

a1.channels.c2.type=file
a1.channels.c2.capacity=1000
a1.channels.c2.transactionCapacity=100

a1.sinks.s1.type=hdfs
a1.sinks.s1.hdfs.path=/flume/%Y/%m/%d/mulc1
a1.sinks.s1.hdfs.filePrefix=s1sink
a1.sinks.s1.hdfs.fileSuffix=.log
a1.sinks.s1.hdfs.inUseSuffix=.tmp
a1.sinks.s1.hdfs.rollInterval=60
a1.sinks.s1.hdfs.rollSize=1024
a1.sinks.s1.hdfs.rollCount=10
a1.sinks.s1.hdfs.idleTimeout=0
a1.sinks.s1.hdfs.batchSize=100
a1.sinks.s1.hdfs.fileType=DataStream
a1.sinks.s1.hdfs.writeFormat=Text
a1.sinks.s1.hdfs.round=true
a1.sinks.s1.hdfs.roundValue=1
a1.sinks.s1.hdfs.roundUnit=second
a1.sinks.s1.hdfs.useLocalTimeStamp=true

a1.sinks.s2.type=hdfs
a1.sinks.s2.hdfs.path=/flume/%Y/%m/%d/mulc2
a1.sinks.s2.hdfs.filePrefix=s2sink
a1.sinks.s2.hdfs.fileSuffix=.log
a1.sinks.s2.hdfs.inUseSuffix=.tmp
a1.sinks.s2.hdfs.rollInterval=60
a1.sinks.s2.hdfs.rollSize=1024
a1.sinks.s2.hdfs.rollCount=10
a1.sinks.s2.hdfs.idleTimeout=0
a1.sinks.s2.hdfs.batchSize=100
a1.sinks.s2.hdfs.fileType=DataStream
a1.sinks.s2.hdfs.writeFormat=Text
a1.sinks.s2.hdfs.round=true
a1.sinks.s2.hdfs.roundValue=1
a1.sinks.s2.hdfs.roundUnit=second
a1.sinks.s2.hdfs.useLocalTimeStamp=true

a1.sources.r1.channels=c1 c2
a1.sinks.s1.channel=c1
a1.sinks.s2.channel=c2

flume-ng agent -c $FLUME_HOME/conf -f /opt/apache-flume-1.6.0-bin/conf/m2.conf -n a1 -Dflume.root.logger=INFO,console

============项目====================

a1.sources = r1
a1.channels = c1
a1.sinks = s1

a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /usr/local/nginx/logs/access.log
a1.sources.r1.interceptors=i1 i2 i3
a1.sources.r1.interceptors.i1.type=timestamp
a1.sources.r1.interceptors.i1.preserveExisting=false
a1.sources.r1.interceptors.i2.type=host
a1.sources.r1.interceptors.i2.preserveExisting=false
a1.sources.r1.interceptors.i2.useIP=true
a1.sources.r1.interceptors.i2.hostHeader=hostname
a1.sources.r1.interceptors.i3.type=static
a1.sources.r1.interceptors.i3.preserveExisting=false
a1.sources.r1.interceptors.i3.key=hn
a1.sources.r1.interceptors.i3.value=hdp

#描述channel 
a1.channels.c1.type=file
a1.channels.c1.capacity=1000                    
a1.channels.c1.transactionCapacity=100           
a1.channels.c1.keep-alive=3
a1.channels.c1.byteCapacityBufferPercentage=20    
a1.channels.c1.byteCapacity=800000

#描述sink
a1.sinks.s1.type=hdfs
a1.sinks.s1.hdfs.path=/flumedemo/%Y/%m/%d/test
a1.sinks.s1.hdfs.filePrefix=s1sink
a1.sinks.s1.hdfs.fileSuffix=.log
a1.sinks.s1.hdfs.inUseSuffix=.tmp
a1.sinks.s1.hdfs.rollInterval=60              //生成新文件
a1.sinks.s1.hdfs.rollSize=1024
a1.sinks.s1.hdfs.rollCount=10
a1.sinks.s1.hdfs.idleTimeout=0
a1.sinks.s1.hdfs.batchSize=100
a1.sinks.s1.hdfs.fileType=DataStream
a1.sinks.s1.hdfs.writeFormat=Text
a1.sinks.s1.hdfs.round=true                   //生成新目录
a1.sinks.s1.hdfs.roundValue=1
a1.sinks.s1.hdfs.roundUnit=second
a1.sinks.s1.hdfs.useLocalTimeStamp=true

#将source和sink绑定到channel
a1.sources.r1.channels = c1
a1.sinks.s1.channel = c1

flume-ng agent -c /opt/apache-flume-1.6.0-bin/conf -f /opt/apache-flume-1.6.0-bin/conf/test.conf -n a1 -Dflume.root.logger=INFO,console

-----kafka示例--------------------------------

a1.sources = r1
a1.sinks = k1
a1.channels = c1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir =/root/my

# 用于描述channel，在内存中做数据的临时的存储
a1.channels.c1.type = memory
a1.channels.c1.capacity=1000                    
a1.channels.c1.transactionCapacity=100           
a1.channels.c1.keep-alive=3
a1.channels.c1.byteCapacityBufferPercentage=20    
a1.channels.c1.byteCapacity=800000

# 用于描述sink，类型是日志格式，用于定制消息发布方的参数
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = recharge
a1.sinks.k1.brokerList = NODE01:9092,NODE02:9092,NODE03:9092

# 将a1中的各个组件建立关联关系，将source和sink都指向了同一个channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

flume-ng agent -c /opt/apache-flume-1.6.0-bin/conf -f /opt/