flume配置文件

最新推荐文章于 2023-04-24 12:34:33 发布

九万里_w

最新推荐文章于 2023-04-24 12:34:33 发布

阅读量776

点赞数

分类专栏： Flume

本文链接：https://blog.csdn.net/hm_rq/article/details/85053523

版权

Flume 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

1.Sink-ES

#------agent-----
agent.sources=agentSource
agent.channels=agentChannel
agent.sinks=agentSink

#-------- agentSource-----------
agent.sources.agentSource.type=org.apache.flume.source.kafka.KafkaSource
agent.sources.agentSource.zookeeperConnect=IP:2181
agent.sources.agentSource.topic=TopicName
agent.sources.agentSource.groupId=Flume
agent.sources.agentSource.batchSize=1000
agent.sources.agentSource.kafka.consumer.timeout.ms=3000

#------- agentChannel-----------
agent.channels.agentChannel.type=file
agent.channels.agentChannel.checkpointDir=/mnt/data/flume/checkpoint
agent.channels.agentChannel.dataDirs=/mnt/data/flume
agent.channels.agentChannel.capacity=100000
agent.channels.agentChannel.transactionCapacity= 1000
agent.channels.agentChannel.keep-alive=100
agent.sources.agentSource.channels=agentChannel
agent.sinks.agentSink.channel=agentChannel

#---------agentSink ------------
agent.sinks.agentSink.type=elasticsearch
agent.sinks.agentSink.hostNames=hostIp:12300
agent.sinks.agentSink.clusterName=es
agent.sinks.agentSink.indexName=indexName
agent.sinks.agentSink.indexType=indexType
agent.sinks.agentSink.idName=ID
agent.sinks.agentSink.batchSize=1000

2.Sink-HBase

#------agent-----
agent.sources=agentSource
agent.channels=agentChannel
agent.sinks=agentSink

#---------agentSink-------------------
agent.sinks.agentSink.type=hbase
agent.sinks.agentSink.zookeeperQuorum=ip:2181
agent.sinks.agentSink.table=tableName
agent.sinks.agentSink.tableType=table
agent.sinks.agentSink.columnFamily=d
agent.sinks.agentSink.serializer=org.apache.flume.sink.hbase.RegexHbaseEventSerializer
agent.sinks.agentSink.serializer.colNames=RowKey
agent.sinks.agentSink.serializer.rowKeyIndex=0
agent.sinks.agentSink.batchSize=1000

3.Sink-HDFS

#------agent-----
agent.sources=agentSource
agent.channels=agentChannel
agent.sinks=agentSink

#---------agentSink------------------
agent.sinks.agentSink.type=hdfs
agent.sinks.agentSink.hdfs.path =/data/flume/partitionday=%Y%m%d/partitionhour=%Y%m%d%H
agent.sinks.agentSink.hdfs.filePrefix=fileName_%Y%m%d%H
agent.sinks.agentSink.hdfs.timeZone=Asia/Shanghai
agent.sinks.agentSink.hdfs.fileType=DataStream
agent.sinks.agentSink.hdfs.batchSize=1000
agent.sinks.agentSink.hdfs.rollCount=0
agent.sinks.agentSink.hdfs.rollInterval=600
agent.sinks.agentSink.hdfs.rollSize=134217728
agent.sinks.agentSink.hdfs.callTimeout=7200000