单节点flume的搭建

1
[hadoop@h91 ~]$ tar -zxvf flume-ng-1.2.0-cdh3u5.tar.gz

[hadoop@h91 ~]$ vi .bash_profile
添加：
export FLUME_HOME=/home/hadoop/flume-ng-1.2.0-cdh3u5
export FLUME_CONF_DIR=$FLUME_HOME/conf
2
[hadoop@h91 ~]$ cd flume-ng-1.2.0-cdh3u5/conf/

[hadoop@h91 conf]$ cp flume-conf.properties.template flume-conf.properties

[hadoop@h91 conf]$ vi flume-conf.properties

a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = hadoop110
a1.sources.r1.port = 44444

# Describe the sink
a1.sinks.k1.type = logger

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

运行flume
3.
[hadoop@h91 flume-ng-1.2.0-cdh3u5]$  bin/flume-ng agent --conf /home/hadoop/flume-ng-1.2.0-cdh3u5/conf/ --conf-file conf/flume-conf.properties --name a1 -Dflume.root.logger=INFO,console

***********
-Dflume.root.logger=INFO,console 仅为 debug 使用，请勿生产环境生搬硬套，否则大量的日志会返回到终端。
-c/--conf 后跟配置目录，-f/--conf-file 后跟具体的配置文件，-n/--name 指定agent的名称
***********

4.另一个界面
[hadoop@h91 ~]$ telnet hadoop 44444
输入 hello world! 
回车

5.在运行 bin/flume-ng 界面能看到结果
urce.start(NetcatSource.java:161)] Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/192.168.8.91:44444]

2015-10-14 12:12:08,114 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 68 65 6C 6C 6F 20 77 6F 72 6C 64 21 0D          hello world!. }

===================================================

单节点 写入到hdfs

agent1.channels = ch1
agent1.sources = avro-source1
agent1.sinks = log-sink1
# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory

#缓存的最大容量
agent1.channels.ch1.capacity = 100000

#每事务的最大容量
agent1.channels.ch1.transactionCapacity = 100000
agent1.channels.ch1.keep-alive = 30

#define source monitor a file
agent1.sources.avro-source1.type = exec
agent1.sources.avro-source1.shell = /bin/bash -c
agent1.sources.avro-source1.command = tail -n +0 -F /home/hadoop/ip.txt
agent1.sources.avro-source1.channels = ch1
agent1.sources.avro-source1.threads = 5

# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
agent1.sinks.log-sink1.channel = ch1
agent1.sinks.log-sink1.type = hdfs
agent1.sinks.log-sink1.hdfs.path = hdfs://192.168.8.110:9000/user/hadoop/flumeTest
agent1.sinks.log-sink1.hdfs.writeFormat = Text
agent1.sinks.log-sink1.hdfs.fileType = DataStream

#hdfs创建多长时间新建文件，0不基于时间
agent1.sinks.log-sink1.hdfs.rollInterval = 0

#基于数据大小  1000字节
agent1.sinks.log-sink1.hdfs.rollSize = 1000

#hdfs有多少条消息时新建文件，0不基于消息个数
agent1.sinks.log-sink1.hdfs.rollCount = 0

agent1.sinks.log-sink1.hdfs.batchSize = 1000
agent1.sinks.log-sink1.hdfs.txnEventMax = 1000
agent1.sinks.log-sink1.hdfs.callTimeout = 60000
agent1.sinks.log-sink1.hdfs.appendTimeout = 60000

 bin/flume-ng agent --conf /home/hadoop/flume-ng-1.2.0-cdh3u5/conf/ --conf-file conf/abc-conf --name agent1 -Dflume.root.logger=INFO,console
/bin/flume-ng agent --conf /home/hadoop/flume-ng-1.2.0-cdh3u5/conf/ -f conf/aaa -n agent1 -Dflume.root.logger=INFO,console

======================
多source
a1.sources = r1 r2 r3 
a1.sinks = k1 
a1.channels = c1 
# 
# Describe/configure the source 
a1.sources.r1.type = exec 
a1.sources.r1.command = tail -F /application/flume/logs/access.log 
  
a1.sources.r2.type = exec 
a1.sources.r2.command = tail -F /application/flume/logs/uqc_head.log 
  
a1.sources.r3.type = exec 
a1.sources.r3.command = tail -F /application/flume/logs/uqc_tail.log 
  
# Describe the sink 
a1.sinks.k1.type = logger 
  
# Use a channel which buffers events in memory 
a1.channels.c1.type = memory 
a1.channels.c1.capacity = 1000 
a1.channels.c1.transactionCapacity = 100 
  
# Bind the source and sink to the channel 
a1.sources.r1.channels = c1 
a1.sources.r2.channels = c1 
a1.sources.r3.channels = c1 
  
a1.sinks.k1.channel = c1