1
[hadoop@hd1 ~]$ tar -zxvf flume-ng-1.6.0-cdh5.5.2.tar.gz -C /home/hadoop
[hadoop@hd1 ~]$ vi .bash_profile
添加:
export FLUME_HOME=/home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin
export FLUME_CONF_DIR=$FLUME_HOME/conf
2
[hadoop@hd1 ~]$ cd apache-flume-1.6.0-cdh5.5.2-bin/conf/
[hadoop@hd1 conf]$ cp flume-conf.properties.template flume-conf.properties
[hadoop@hd1 conf]$ vi flume-conf.properties
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = hd1
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
运行flume
bin/flume-ng agent --conf /home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin/conf/ --conf-file conf/flume-conf.properties --name a1 -Dflume.root.logger=INFO,console
***********
-Dflume.root.logger=INFO,console 仅为 debug 使用,请勿生产环境生搬硬套,否则大量的日志会返回到终端。
-c/--conf 后跟配置目录,-f/--conf-file 后跟具体的配置文件,-n/--name 指定agent的名称
***********
4.另一个界面
[hadoop@hd1 ~]$ telnet hd1 44444
5.在运行 bin/flume-ng 界面能看到结果
urce.start(NetcatSource.java:161)] Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.8.91:44444]
2015-10-14 12:12:08,114 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 68 65 6C 6C 6F 20 77 6F 72 6C 64 21 0D hello world!. }
===================================================
单节点 写入到hdfs
agent1.channels = ch1
agent1.sources = avro-source1
agent1.sinks = log-sink1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
#缓存的最大容量
agent1.channels.ch1.capacity = 100000
#每事务的最大容量
agent1.channels.ch1.transactionCapacity = 10000
agent1.channels.ch1.keep-alive = 30
#define source monitor a file
agent1.sources.avro-source1.type = exec
agent1.sources.avro-source1.shell = /bin/bash -c
agent1.sources.avro-source1.command = tail -n +0 -F /home/hadoop/flume.txt
#指定抽取的文件
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.threads = 5
# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
agent1.sinks.log-sink1.channel = ch1
agent1.sinks.log-sink1.type = hdfs
agent1.sinks.log-sink1.hdfs.path = hdfs://172.16.1.138:9000/user/hadoop/had0320
agent1.sinks.log-sink1.hdfs.writeFormat = Text
agent1.sinks.log-sink1.hdfs.fileType = DataStream
#hdfs创建多长时间新建文件,0不基于时间
agent1.sinks.log-sink1.hdfs.rollInterval = 0
#基于数据大小 1000字节
agent1.sinks.log-sink1.hdfs.rollSize = 1000
#hdfs有多少条消息时新建文件,0不基于消息个数
agent1.sinks.log-sink1.hdfs.rollCount = 0
agent1.sinks.log-sink1.hdfs.batchSize = 1000
agent1.sinks.log-sink1.hdfs.txnEventMax = 1000
agent1.sinks.log-sink1.hdfs.callTimeout = 60000
agent1.sinks.log-sink1.hdfs.appendTimeout = 60000
bin/flume-ng agent --conf /home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin/conf/ --conf-file conf/bb.properties --name agent1 -Dflume.root.logger=INFO,console
/bin/flume-ng agent --conf /home/hadoop/flume-ng-1.2.0-cdh3u5/conf/ -f conf/flume -n agent1 -Dflume.root.logger=INFO,console
======================
多source
a1.sources = r1 r2 r3
a1.sinks = k1
a1.channels = c1
#
# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /application/flume/logs/access.log
a1.sources.r2.type = exec
a1.sources.r2.command = tail -F /application/flume/logs/uqc_head.log
a1.sources.r3.type = exec
a1.sources.r3.command = tail -F /application/flume/logs/uqc_tail.log
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sources.r2.channels = c1
a1.sources.r3.channels = c1
a1.sinks.k1.channel = c1
flume的搭建
最新推荐文章于 2024-07-18 21:37:27 发布