第一步:解压缩
[root@service_es3 conf]
第二步:修改conf的env文件
[root@service_es3 software]
[root@service_es3 conf]
export JAVA_HOME=/usr/local/java
export JAVA_OPTS="-Xms256m -Xmx1G -Dcom.sun.management.jmxremote"
第三步:添加flume日志采集配置文件
[root@service_es3 apache-flume-1.6.0-cdh5.11.0-bin]# mkdir appConf
### define agent
bdpAgent.sources = r3
bdpAgent.channels = c3
bdpAgent.sinks = k3
### define sources
bdpAgent.sources.r3.type = TAILDIR
bdpAgent.sources.r3.spoolDir = /opt/data/gps
bdpAgent.sources.r3.positionFile = /opt/software/flume-1.6.0-cdh5.11.0/source/gps/taildir_position.json
bdpAgent.sources.r3.filegroups = f1
bdpAgent.sources.r3.filegroups.f1 = /opt/data/gps/.*.complete
bdpAgent.sources.r3.headers.f1.headerKey1 = value1
bdpAgent.sources.r3.fileHeader = true
### define channels
bdpAgent.channels.c3.type = file
bdpAgent.channels.c3.checkpointDir = /opt/software/flume-1.6.0-cdh5.11.0/channels/gps/checkpointDir
bdpAgent.channels.c3.dataDirs = /opt/software/flume-1.6.0-cdh5.11.0/channels/gps/dataDir
### define sink
bdpAgent.sinks.k3.type = hdfs
bdpAgent.sinks.k3.hdfs.path = hdfs://10.27.227.160:8020/xqtravel-boss-bd/bufferData/gpsDetail
bdpAgent.sinks.k3.hdfs.fileType = DataStream
bdpAgent.sinks.k3.hdfs.writeFormat = Text
bdpAgent.sinks.k3.hdfs.useLocalTimeStamp = true
bdpAgent.sinks.k3.hdfs.rollInterval = 14400
bdpAgent.sinks.k3.hdfs.rollSize = 0
bdpAgent.sinks.k3.hdfs.rollCount = 0
bdpAgent.sinks.k3.hdfs.idleTimeout = 0
bdpAgent.sinks.k3.hdfs.filePrefix = %Y%m%d_
bdpAgent.sinks.k3.hdfs.fileSuffix = .log
### bind the soures and sink to the channel
bdpAgent.sources.r3.channels = c3
bdpAgent.sinks.k3.channel = c3
第四步:添加flume依赖的jar包
commons-configuration-1.6.jar
commons-daemon-1.0.13.jar
hadoop-auth-2.6.0.jar
hadoop-common-2.6.0-cdh5.11.0.jar
hadoop-hdfs-2.6.0-cdh5.11.0.jar
hadoop-hdfs-nfs-2.6.0-cdh5.11.0.jar
hadoop-mapreduce-client-core-2.6.0-cdh5.11.0.jar
hadoop-nfs-2.6.0-cdh5.11.0.jar
htrace-core4-4.0.1-incubating.jar
第五步:设置flume日志收集主程序
[root@service_es3 apache-flume-1.6.0-cdh5.11.0-bin]
[root@es flume-ng-1.6.0-cdh5.11.0]
#! /bin/bash
nohup ../bin/flume-ng agent -n bdpAgent -c ../conf -f ../appConf/flume-xqtravelbdp-gpsDetail.conf -Dflume.root.logger=INFO,console > gpsDetailCollector.log &