02-taildir source

------------------------------------taildir source ------------------------------------
1、内容查询

====================================
[root@singlelucky flume]# head -10000 prolog.log>tail01/prologhead10000.log
[root@singlelucky flume]# ls tail01
prologhead10000.log
[root@singlelucky flume]# wc tail01/prologhead10000.log 
  10000   50000   1688953 tail01/prologhead10000.log
  行数	  单词数	大小
====================================

2、创建cnf

vim test_taildir_hffs.cnf

3、配置信息

# 组件说明
a1.sources 	= s1
a1.channels = c1
a1.sinks    = k1

# 初始化数据源
a1.sources.s1.type = taildir
a1.sources.s1.filegroups = f1 f2
a1.sources.s1.filegroups.f1 = /root/data/flume/tail01/prolog.*\\.log
a1.sources.s1.filegroups.f2 = /root/data/flume/tail02/prolog.*\\.log
a1.sources.s1.positionFile = /opt/software/flume190/data/taildir/taildir_position.conf

# 初始化通道
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /opt/software/flume190/mydata/checkpoint
a1.channels.c1.dataDirs = /opt/software/flume190/mydata/data 
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 1000

# 初始化数据槽
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.19.130:9820/flume/events/tailevent/%Y-%m-%d/%H
a1.sinks.k1.hdfs.round = true		# 时间舍弃
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.filePrefix = log_%Y%m%d_%H
a1.sinks.k1.hdfs.fileSuffix = .log
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 134217728
a1.sinks.k1.hdfs.bacthSize = 1000
a1.sinks.k1.hdfs.threadsPoolSize = 4
a1.sinks.k1.hdfs.idleTimeout = 0
a1.sinks.k1.hdfs.minBlockReplicas = 1

# 关联组件
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

4、启动

flume-ng agent -n a1 -c conf/ -f flume-conf-files/test_taildir_hffs.cnf -Dflume.root.logger=INFO,console