Flume-拦截器interceptors

Flume-拦截器interceptors

一、timestamp和hostname拦截器

#set agent name
a1.sources=r1
a1.channels=c1
a1.sinks=k1

#set channels
a1.channels.c1.type=memory
a1.channels.c1.capacity=10000
a1.channels.c1.transactionCapacity=1000

#set sources
a1.sources.r1.type=avro
a1.sources.r1.bind=0.0.0.0 #任意一个主机的IP地址
a1.sources.r1.port=52013
a1.sources.r1.channels=c1

#set interceptors
a1.sources.r1.interceptors=i1 i2
a1.sources.r1.interceptors.i1.type=timestamp
a1.sources.r1.interceptors.i2.type=hostname
a1.sources.r1.interceptors.i2.hostHeader=hostname
a1.sources.r1.interceptors.i2.useIP=false #将IP地址转换为hostname主机名

#set sinks
a1.sinks.k1.type=hdfs
a1.sinks.k1.channel=c1
a1.sinks.k1.hdfs.path=/%{hostname}
a1.sinks.k1.hdfs.filePrefix=%Y-%m-%d
a1.sinks.k1.hdfs.round=true
a1.sinks.k1.hdfs.roundValue=10
a1.sinks.k1.hdfs.roundUnit=minute
a1.sinks.k1.hdfs.rollInterval=60
a1.sinks.k1.hdfs.rollSize=50
a1.sinks.k1.hdfs.rollCount=100
a1.sinks.k1.hdfs.batchSize=1000
a1.sinks.k1.hdfs.fileType=DataStream

二、static静态自定义（K,V）拦截器

#********************* set agent name ****************
a1.sources=r1 r2 r3
a1.channels=c1
a1.sinks=k1

#********************* set sources ***********************
#set source r1
a1.sources.r1.type=exec
a1.sources.r1.command=tail -F access.log
a1.sources.r1.interceptors=i1
a1.sources.r1.interceptors.i1.type=static  #往采集数据的header中插入（K,V）键值对
a1.sources.r1.interceptors.i1.key=type
a1.sources.r1.ingerceptors.i1.value=access

#set source r2
a1.sources.r2.type=exec
a1.sources.r2.command=tail -F nginx.log
a1.sources.r2.interceptors=i2
a1.sources.r2.interceptors.i2.type=static
a1.sources.r2.interceptors.i2.key=type
a1.sources.r2.interceptors.i2.value=nginx

#set source r3
a1.sources.r3.type=exec
a1.sources.r3.command=tail -F web.log
a1.sources.r3.interceptors=i3
a1.sources.r3.interceptors.i3.type=static
a1.sources.r3.interceptors.i3.key=type
a1.sources.r3.interceptors.i3.value=web

a1.sources.r1.channels=c1
a1.sources.r2.channels=c1
a1.sources.r3.channels=c1

#******************* set channels ******************
a1.channels.c1.type=memory
a1.channels.c1.capacity=100000
a1.channels.c1.transactionCapacity=1000

#**************** set sinks ********************
a1.sinks.k1.type=avro
a1.sinks.k1.channel=c1
a1.sinks.k1.hostname=192.168.200.10
a1.sinks.k1.port=52013