flume1.7.0新功能taildir日志采集

第一步：解压缩

[root@service_es3 conf]# tar -zvx -f flume-ng-1.6.0-cdh5.11.0.tar.gz 

第二步：修改conf的env文件

[root@service_es3 software]# cd apache-flume-1.6.0-cdh5.11.0-bin/conf/
[root@service_es3 conf]# cp -a flume-env.sh.template flume-env.sh
export JAVA_HOME=/usr/local/java
export JAVA_OPTS="-Xms256m -Xmx1G -Dcom.sun.management.jmxremote"

第三步：添加flume日志采集配置文件

[root@service_es3 apache-flume-1.6.0-cdh5.11.0-bin]# mkdir appConf 

### define agent
bdpAgent.sources = r3
bdpAgent.channels = c3
bdpAgent.sinks = k3

### define sources
bdpAgent.sources.r3.type = TAILDIR
bdpAgent.sources.r3.spoolDir = /opt/data/gps
bdpAgent.sources.r3.positionFile = /opt/software/flume-1.6.0-cdh5.11.0/source/gps/taildir_position.json
bdpAgent.sources.r3.filegroups = f1
bdpAgent.sources.r3.filegroups.f1 = /opt/data/gps/.*.complete
bdpAgent.sources.r3.headers.f1.headerKey1 = value1
bdpAgent.sources.r3.fileHeader = true

### define channels
bdpAgent.channels.c3.type = file
bdpAgent.channels.c3.checkpointDir = /opt/software/flume-1.6.0-cdh5.11.0/channels/gps/checkpointDir
bdpAgent.channels.c3.dataDirs = /opt/software/flume-1.6.0-cdh5.11.0/channels/gps/dataDir

### define sink
bdpAgent.sinks.k3.type = hdfs
bdpAgent.sinks.k3.hdfs.path = hdfs://10.27.227.160:8020/xqtravel-boss-bd/bufferData/gpsDetail
bdpAgent.sinks.k3.hdfs.fileType = DataStream
bdpAgent.sinks.k3.hdfs.writeFormat = Text
bdpAgent.sinks.k3.hdfs.useLocalTimeStamp = true
bdpAgent.sinks.k3.hdfs.rollInterval = 14400
bdpAgent.sinks.k3.hdfs.rollSize = 0
bdpAgent.sinks.k3.hdfs.rollCount = 0
bdpAgent.sinks.k3.hdfs.idleTimeout = 0
bdpAgent.sinks.k3.hdfs.filePrefix = %Y%m%d_
bdpAgent.sinks.k3.hdfs.fileSuffix = .log

### bind the soures and sink to the channel
bdpAgent.sources.r3.channels = c3
bdpAgent.sinks.k3.channel = c3

第四步：添加flume依赖的jar包

commons-configuration-1.6.jar  
commons-daemon-1.0.13.jar  
hadoop-auth-2.6.0.jar   
hadoop-common-2.6.0-cdh5.11.0.jar  
hadoop-hdfs-2.6.0-cdh5.11.0.jar  
hadoop-hdfs-nfs-2.6.0-cdh5.11.0.jar  
hadoop-mapreduce-client-core-2.6.0-cdh5.11.0.jar  
hadoop-nfs-2.6.0-cdh5.11.0.jar  
htrace-core4-4.0.1-incubating.jar 

第五步：设置flume日志收集主程序

[root@service_es3 apache-flume-1.6.0-cdh5.11.0-bin]# mkdir appPro
[root@es flume-ng-1.6.0-cdh5.11.0]# cd appPro/
#! /bin/bash
nohup ../bin/flume-ng agent -n bdpAgent -c ../conf -f ../appConf/flume-xqtravelbdp-gpsDetail.conf  -Dflume.root.logger=INFO,console > gpsDetailCollector.log &