监视文件夹
启动命令:
bin/flume-ng agent -c ./conf -f ./conf/spool-hdfs.conf -n a1 -Dflume.root.logger=INFO,console
测试: 往/home/hadoop/flumeSpool放文件(mv ././xxxFile /home/hadoop/flumeSpool),但是不要在里面生成文件
配置
a1.sources = r1
a1.sinks = k1
a1.channels = c1
##注意:不能往监控目中重复丢同名文件
a1.sources.r1.type = spooldir (cdh) /exec(cdh) /taildir(apache1.7版本之后,监控文件和目录,断点续查)
a1.sources.r1.spoolDir = /root/logs2 //监控的文件夹
a1.sources.r1.fileHeader = true
a1.sinks.k1.type = hdfs
a1.sinks.k1.channel = c1
a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/ //上传hdfs的路径
a1.sinks.k1.hdfs.filePrefix = events- //加个前缀
a1.sinks.k1.hdfs.round = true //是符开启时间上的舍弃
a1.sinks.k1.hdfs.roundValue = 10 //10分钟数据滚动,10分钟创建一个文件夹
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.rollInterval = 0 //以间隔时间滚动
a1.sinks.k1.hdfs.rollSize = 134217728 //以文件大小滚动 byte
a1.sinks.k1.hdfs.rollCount = 0 //以event的个数滚动
a1.sinks.k1.hdfs.idleTimeout= 0 //超时时间设置
a1.sinks.k1.hdfs.batchSize = 1 //1批次写几个
a1.sinks.k1.hdfs.useLocalTimeStamp = true //开启本地时间戳功能
#生成的文件类型,默认是Sequencefile,可用DataStream,则为普通文本
a1.sinks.k1.hdfs.fileType = DataStream
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1