flume的搭建

1
[hadoop@hd1 ~]$ tar -zxvf flume-ng-1.6.0-cdh5.5.2.tar.gz -C /home/hadoop 

[hadoop@hd1 ~]$ vi .bash_profile
添加：
export FLUME_HOME=/home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin
export FLUME_CONF_DIR=$FLUME_HOME/conf
2
[hadoop@hd1 ~]$ cd apache-flume-1.6.0-cdh5.5.2-bin/conf/

[hadoop@hd1 conf]$ cp flume-conf.properties.template flume-conf.properties

[hadoop@hd1 conf]$ vi flume-conf.properties

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = hd1
a1.sources.r1.port = 44444

# Describe the sink
a1.sinks.k1.type = logger

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1



运行flume
bin/flume-ng agent --conf /home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin/conf/ --conf-file conf/flume-conf.properties --name a1 -Dflume.root.logger=INFO,console

***********
-Dflume.root.logger=INFO,console 仅为 debug 使用，请勿生产环境生搬硬套，否则大量的日志会返回到终端。
-c/--conf 后跟配置目录，-f/--conf-file 后跟具体的配置文件，-n/--name 指定agent的名称
***********

4.另一个界面
[hadoop@hd1 ~]$ telnet hd1 44444

5.在运行 bin/flume-ng 界面能看到结果
urce.start(NetcatSource.java:161)] Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.8.91:44444]

2015-10-14 12:12:08,114 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 68 65 6C 6C 6F 20 77 6F 72 6C 64 21 0D          hello world!. }



===================================================

单节点 写入到hdfs

agent1.channels = ch1
agent1.sources = avro-source1
agent1.sinks = log-sink1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory

#缓存的最大容量
agent1.channels.ch1.capacity = 100000

#每事务的最大容量
agent1.channels.ch1.transactionCapacity = 10000
agent1.channels.ch1.keep-alive = 30


#define source monitor a file
agent1.sources.avro-source1.type = exec
agent1.sources.avro-source1.shell = /bin/bash -c
agent1.sources.avro-source1.command = tail -n +0 -F /home/hadoop/flume.txt
#指定抽取的文件
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.threads = 5

# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
agent1.sinks.log-sink1.channel = ch1
agent1.sinks.log-sink1.type = hdfs
agent1.sinks.log-sink1.hdfs.path = hdfs://172.16.1.138:9000/user/hadoop/had0320
agent1.sinks.log-sink1.hdfs.writeFormat = Text
agent1.sinks.log-sink1.hdfs.fileType = DataStream

#hdfs创建多长时间新建文件，0不基于时间
agent1.sinks.log-sink1.hdfs.rollInterval = 0

#基于数据大小  1000字节
agent1.sinks.log-sink1.hdfs.rollSize = 1000

#hdfs有多少条消息时新建文件，0不基于消息个数
agent1.sinks.log-sink1.hdfs.rollCount = 0

agent1.sinks.log-sink1.hdfs.batchSize = 1000
agent1.sinks.log-sink1.hdfs.txnEventMax = 1000
agent1.sinks.log-sink1.hdfs.callTimeout = 60000
agent1.sinks.log-sink1.hdfs.appendTimeout = 60000


 bin/flume-ng agent --conf /home/hadoop/apache-flume-1.6.0-cdh5.5.2-bin/conf/ --conf-file conf/bb.properties --name agent1 -Dflume.root.logger=INFO,console 

/bin/flume-ng agent --conf /home/hadoop/flume-ng-1.2.0-cdh3u5/conf/ -f conf/flume -n agent1 -Dflume.root.logger=INFO,console




======================
多source
a1.sources = r1 r2 r3  
a1.sinks = k1  
a1.channels = c1  
#  
# Describe/configure the source  
a1.sources.r1.type = exec  
a1.sources.r1.command = tail -F /application/flume/logs/access.log  
   
a1.sources.r2.type = exec  
a1.sources.r2.command = tail -F /application/flume/logs/uqc_head.log  
   
a1.sources.r3.type = exec  
a1.sources.r3.command = tail -F /application/flume/logs/uqc_tail.log  
   
# Describe the sink  
a1.sinks.k1.type = logger  
   
# Use a channel which buffers events in memory  
a1.channels.c1.type = memory  
a1.channels.c1.capacity = 1000 
a1.channels.c1.transactionCapacity = 100  
   
# Bind the source and sink to the channel  
a1.sources.r1.channels = c1  
a1.sources.r2.channels = c1  
a1.sources.r3.channels = c1  
   
a1.sinks.k1.channel = c1