flume从kafka获取数据并按时间保存到hdfs上

最新推荐文章于 2024-03-09 23:16:05 发布

doulejian

最新推荐文章于 2024-03-09 23:16:05 发布

阅读量6k

点赞数 2

分类专栏： flume 文章标签：数据 hdfs flume kafka

本文链接：https://blog.csdn.net/doulejian/article/details/56009112

版权

flume 专栏收录该内容

0 篇文章 0 订阅

订阅专栏

#############################flume-config-name 从kafka获取数据并保存到hdfs上###########################
#source名称
flume-config-name.sources = source-flume-config
#channel名称
flume-config-name.channels = channel-flume-config
#sink名称
flume-config-name.sinks = sink-flume-config

#source flume-config 配置source
#配置kafka信息
flume-config-name.sources.source-flume-config.type = org.apache.flume.source.kafka.KafkaSource
#zk地址，多个以逗号分开
flume-config-name.sources.source-flume-config.zookeeperConnect = localhost:2181
#配置topic名称以及groupid
flume-config-name.sources.source-flume-config.topic = flume-topic
flume-config-name.sources.source-flume-config.groupId = flumev01
#以下为配置参数
flume-config-name.sources.source-flume-config.batchSize = 20000
flume-config-name.sources.source-flume-config.batchDurationMillis = 1000
flume-config-name.sources.source-flume-config.backoffSleepIncrement = 1000
flume-config-name.sources.source-flume-config.maxBackoffSleep = 5000
flume-config-name.sources.source-flume-config.kafka.consumer.timeout.ms = 100

#sink-flume-config
#sink类型为hdfs
flume-config-name.sinks.sink-flume-config.type = hdfs
#配置保存地址，需有flume的读写权限，设置chown flume:hdfs /test/test , 按照时间存储直接按以下配置
flume-config-name.sinks.sink-flume-config.hdfs.path = /test/test/%Y-%m-%d/%H
#文件前缀
flume-config-name.sinks.sink-flume-config.hdfs.filePrefix = test
#如果文件正在占用，后缀为.tmp.gz
flume-config-name.sinks.sink-flume-config.hdfs.inUseSuffix = .tmp.gz
#正常文件后缀.log.gz
flume-config-name.sinks.sink-flume-config.hdfs.fileSuffix = .log.gz
#以下为相关配置
flume-config-name.sinks.sink-flume-config.hdfs.useLocalTimeStamp = true
flume-config-name.sinks.sink-flume-config.hdfs.rollInterval = 0
flume-config-name.sinks.sink-flume-config.hdfs.rollSize = 1073741824
flume-config-name.sinks.sink-flume-config.hdfs.rollCount = 0
flume-config-name.sinks.sink-flume-config.hdfs.batchSize = 20000
flume-config-name.sinks.sink-flume-config.hdfs.fileType = CompressedStream
flume-config-name.sinks.sink-flume-config.hdfs.codeC = gzip
flume-config-name.sinks.sink-flume-config.hdfs.idleTimeout = 300
flume-config-name.sinks.sink-flume-config.hdfs.threadsPoolSize = 10
flume-config-name.sinks.sink-flume-config.hdfs.callTimeout = 60000

#channel-flume-config
#配置channel类型
flume-config-name.channels.channel-flume-config.type = memory
#以下两个参数配置可根据实际情况配置
flume-config-name.channels.channel-flume-config.capacity=50000000
flume-config-name.channels.channel-flume-config.transactionCapacity=800000

#配置source的channel , 注意配置项为channels
flume-config-name.sources.source-flume-config.channels = channel-flume-config
#配置sink的channel ，注意配置项为channel
flume-config-name.sinks.sink-flume-config.channel = channel-flume-config