flume常用配置



-----------------------------------example.conf--------------------------------------------
a1.sources = r1
a1.sinks = k1
a1.channels = c1
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
a1.sinks.k1.type = logger
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

----------------------------------启动flume------------------------------------------------
sh bin/flume-ng agent --conf conf --conf-file conf/example.conf --name a1 -Dflume.root.logger=INFO,console

----------------------------------再启动一个终端-------------------------------------------
telnet localhost 44444

----------------------------------flume.conf-----------------------------------------------
# 指定Agent的组件名称
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# 指定Flume source(要监听的路径)
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/omm/app/path

# 指定Flume sink
a1.sinks.k1.type = logger

# 指定Flume channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# 绑定source和sink到channel上
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

----------------------------------启动flume------------------------------------------------
sh bin/flume-ng agent --conf conf --conf-file conf/flume.conf --name a1 -Dflume.root.logger=INFO,console

----------------------------------再启动一个终端-------------------------------------------
在/home/omm/app/path下新增文件

--------------------------------flume2.conf--------------------------------------------------
a1.sources = r1
a1.sinks = k1
a1.channels = c1

#a1.sources.r1.type=exec
#a1.sources.r1.command=tail -F /home/omm/app/path/abc.log
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/omm/app/path
a1.sources.r1.channels=c1

a1.channels.c1.type=memory
a1.channels.c1.capacity=10000
a1.channels.c1.transactionCapacity=100

#设置Kafka接收器
a1.sinks.k1.type= org.apache.flume.sink.kafka.KafkaSink
#设置Kafka的broker地址和端口号
a1.sinks.k1.kafka.bootstrap.servers=99.12.86.173:9092
#设置Kafka的Topic
a1.sinks.k1.kafka.topic=VC_M_T
#设置序列化方式
a1.sinks.k1.serializer.class=kafka.serializer.StringEncoder

a1.sinks.k1.channel=c1

----------------------------------启动flume------------------------------------------------
sh bin/flume-ng agent -n a1 -c conf -f conf/flume2.conf -Dflume.root.logger=INFO,console

----------------------------------avro-----------------------------------------------------
多个agent连接，用avro