flume监听日志存入kafka，hdfs

配置文件：
存入hdfs :

#Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
#Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = 监听的文件路径(在flume启动之前创建)
a1.sources.r1.fileHeader = true
a1.sources.r1.deserializer=LINE
a1.sources.r1.deserializer.maxLineLength=2048(每一行的最大长度)
a1.sources.r1.deserializer.outputCharset=UTF-8

a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type= host
a1.sources.r1.interceptors.i2.type= 自定义的拦截器名称$Builder(拦截器jar包放在lib目录下)

#Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://nodename:port/dirpath(存放到hdfs下的目录路径)
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 10
a1.sinks.k1.hdfs.rollSize = 0
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.filePrefix = %Y-%m-%d-%H-%M-%S（在hdfs上生成的文件名的格式）
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.callTimeout=100000

#Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

存入kafka配置:

#Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

#Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = 监听的文件路径
a1.sources.r1.fileHeader = true

a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type= host
a1.sources.r1.interceptors.i2.type=自定义的拦截器名称$Builder(拦截器jar包放在lib目录下)

#Describe the sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = topicname(主题名)
a1.sinks.k1.kafka.bootstrap.servers = node1:9092,node2:9092,node3:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
a1.sinks.k1.kafka.producer.compression.type = snappy

#Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

同时存入kafka和hdfs:
合并上两个文件即可

# Name the components on this agent
a1.sources = r1
a1.sinks = k1 k2 (定义两个sinks)
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = 监听的文件路径
a1.sources.r1.fileHeader = true
a1.sources.r1.deserializer=LINE
a1.sources.r1.deserializer.maxLineLength=2048
a1.sources.r1.deserializer.outputCharset=UTF-8

a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type= host
a1.sources.r1.interceptors.i2.type= 自定义的拦截器名称$Builder

# Describe the sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://nodename:port/dirpath(存放到hdfs下的目录路径)
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 10
a1.sinks.k1.hdfs.rollSize = 0
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.filePrefix = %Y-%m-%d-%H-%M-%S
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.callTimeout=100000

a1.sinks.k2.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k2.kafka.topic = topicname(主题名)
a1.sinks.k2.kafka.bootstrap.servers = node1:9092,node2:9092,node3:9092
a1.sinks.k2.kafka.flumeBatchSize = 20
a1.sinks.k2.kafka.producer.acks = 1
a1.sinks.k2.kafka.producer.linger.ms = 1
a1.sinks.k2.kafka.producer.compression.type = snappy

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c1

flume启动命令
在bin目录下:

./flume-ng agent --conf conf --conf-file configfilepath --name sourcesname -Dflume.root.logger=DEBUG,console