3、flume数据导入到Hdfs中

[root@baozi apache-flume-1.5.2-bin]# vim conf/agent2.conf


agent2.sources=source1
agent2.channels=channel1
agent2.sinks=sink1


agent2.sources.source1.type=avro
agent2.sources.source1.bind=0.0.0.0
agent2.sources.source1.port=44444
agent2.sources.source1.channels=channel1


agent2.sources.source1.interceptors = i1 i2
agent2.sources.source1.interceptors.i1.type = org.apache.flume.interceptor.HostInterceptor$Builder
agent2.sources.source1.interceptors.i1.preserveExisting = true
agent2.sources.source1.interceptors.i1.useIP = true
agent2.sources.source1.interceptors.i2.type = org.apache.flume.interceptor.TimestampInterceptor$Builder


agent2.channels.channel1.type=memory
agent2.channels.channel1.capacity=10000
agent2.channels.channel1.transactionCapacity=1000
agent2.channels.channel1.keep-alive=30


agent2.sinks.sink1.type=hdfs
agent2.sinks.sink1.channel=channel1
agent2.sinks.sink1.hdfs.path=hdfs://192.168.1.200:9000/flume/events/%{host}/%y-%m-%d
agent2.sinks.sink1.hdfs.fileType=DataStream
agent2.sinks.sink1.hdfs.writeFormat=Text
agent2.sinks.sink1.hdfs.rollInterval=0
agent2.sinks.sink1.hdfs.rollSize=10000
agent2.sinks.sink1.hdfs.rollCount=0
agent2.sinks.sink1.hdfs.idleTimeout=5


启动44444端口：
数据导入到HDFS中这个端口要一直启动：
[root@baozi apache-flume-1.5.2-bin]# bin/flume-ng agent --conf ./conf/ -Dflume.monitoring.type=http -Dflumetoring.port=34343 -n agent2 -f conf/agent2.conf &


log4j.properties：
#log4j.appender.flume.Port = 41414
log4j.appender.flume.Port = 44444



HDFS上的目录/flume/events：
[root@baozi hadoop]# hdfs dfs -ls -R /flume
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-0.99.2/lib/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
drwxr-xr-x   - root supergroup          0 2015-06-21 12:50 /flume/events
[root@baozi hadoop]#



运行代码：
package flume;

import java.text.SimpleDateFormat;
import org.apache.log4j.Logger;

public class LogProducer {
public static void main(String[] args) {
Logger log=Logger.getLogger(LogProducer.class);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
while (true) {
log.info("日志格式："+sdf.format(System.currentTimeMillis()));
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}




查看HDFS里的数据：

[root@baozi hadoop]# hdfs dfs -ls -R /flume/events
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-0.99.2/lib/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
drwxr-xr-x   - root supergroup          0 2015-06-21 12:55 /flume/events/192.168.1.200
drwxr-xr-x   - root supergroup          0 2015-06-21 12:55 /flume/events/192.168.1.200/15-06-21
-rw-r--r--   1 root supergroup          0 2015-06-21 12:55 /flume/events/192.168.1.200/15-06-21/FlumeData.1434862524993.tmp
[root@baozi hadoop]#



oot@baozi hadoop]# hdfs dfs -ls -R /flume/events/192.168.1.200/15-06-21
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-0.99.2/lib/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
-rw-r--r--   1 root supergroup       2940 2015-06-21 12:56 /flume/events/192.168.1.200/15-06-21/FlumeData.1434862524993
[root@baozi hadoop]#


[root@baozi hadoop]# hdfs dfs -ls -R /flume/events
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-0.99.2/lib/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
drwxr-xr-x   - root supergroup          0 2015-06-21 12:55 /flume/events/192.168.1.200
drwxr-xr-x   - root supergroup          0 2015-06-21 12:56 /flume/events/192.168.1.200/15-06-21
-rw-r--r--   1 root supergroup       2940 2015-06-21 12:56 /flume/events/192.168.1.200/15-06-21/FlumeData.1434862524993
[root@baozi hadoop]# hdfs dfs -text /flume/events/192.168.1.200/15-06-21/FlumeData.1434862524993
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/usr/local/hbase-0.99.2/lib/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
日志格式：2015-06-21 12:52:12
日志格式：2015-06-21 12:52:13
日志格式：2015-06-21 12:52:14
日志格式：2015-06-21 12:52:15
日志格式：2015-06-21 12:52:16
日志格式：2015-06-21 12:52:17
日志格式：2015-06-21 12:52:18
日志格式：2015-06-21 12:52:19

[root@baozi hadoop]#