华为 FunsionInsight HD 平台的 配置 source-taildir sink-hdfs

上传为  properties.properties 文件

 

server.sources = serverSource
server.channels = serverChannel
server.sinks = serverSink

server.sources.serverSource.type = TAILDIR
server.sources.serverSource.filegroups = serverSourceGroup
server.sources.serverSource.filegroups.serverSourceGroup = /opt/software/oracle-kafka-opengauss/data/data.txt
server.sources.serverSource.headers.serverSourceGroup. = 
server.sources.serverSource.positionFile = /home/data_log.json
server.sources.serverSource.montime = 
server.sources.serverSource.byteOffsetHeader = false
server.sources.serverSource.skipToEnd = false
server.sources.serverSource.idleTimeout = 12000
server.sources.serverSource.writePosInterval = 3000
server.sources.serverSource.batchSize = 1000
server.sources.serverSource.fileHeader = false
server.sources.serverSource.fileHeaderKey = file
server.sources.serverSource.channels = serverChannel

server.channels.serverChannel.type = memory
server.channels.serverChannel.capacity = 1000000
server.channels.serverChannel.transactionCapacity = 100000


server.sinks.serverSink.type = hdfs
server.sinks.serverSink.hdfs.path = hdfs://hacluster/user/hive/warehouse/on_log
server.sinks.serverSink.hdfs.filePrefix = log_%Y%m%d_%H
server.sinks.serverSink.hdfs.fileSuffix = .log
server.sinks.serverSink.hdfs.idleTimeout = 0
server.sinks.serverSink.hdfs.batchSize = 10000
server.sinks.serverSink.hdfs.codeC =  
server.sinks.serverSink.hdfs.fileType = DataStream
server.sinks.serverSink.hdfs.maxOpenFiles = 5000
server.sinks.serverSink.hdfs.writeFormat = Text
server.sinks.serverSink.hdfs.callTimeout = 7200000
server.sinks.serverSink.hdfs.threadsPoolSize = 10
server.sinks.serverSink.hdfs.kerberosPrincipal = poc_zzw
server.sinks.serverSink.hdfs.kerberosKeytab = /home/poc_zzw/user.keytab
server.sinks.serverSink.hdfs.round = true
server.sinks.serverSink.hdfs.roundUnit = minute
server.sinks.serverSink.hdfs.useLocalTimeStamp = true
server.sinks.serverSink.hdfs.fileCloseByEndEvent = false （这个不改，文件一直为 临时文件）
server.sinks.serverSink.hdfs.rollInterval = 5
server.sinks.serverSink.hdfs.rollSize = 102400000
server.sinks.serverSink.hdfs.rollCount = 0
server.sinks.serverSink.hdfs.roundValue = 10

server.sinks.serverSink.channel = serverChannel