Flume配置案例(三)-Kafka篇

 官方文档（中文）

Flume 1.9用户手册中文版 — 可能是目前翻译最完整的版本了

官方文档（英文）

Flume 1.9.0 User Guide — Apache Flume

1、taildir-kafka

创建flume-taildir-kafka.properties

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
​
# Describe/configure the source
a1.sources.r1.type = TAILDIR
# 通过json的方式，记录了每个文件最新的消费位置
a1.sources.r1.positionFile = /usr/local/soft/apache-flume-1.9.0-bin/log/taildir_position.json
# 指定要检测的目录文件
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1 = /usr/local/logs/crawler/.*logs
a1.sources.r1.fileHeader = true
​
# Describe the sink
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = hadoop100:9092
a1.sinks.k1.kafka.topic = app-crawler
​
# Describe the channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
​
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
​

启动

cd /usr/local/soft/kafka_2.11-2.3.0/
bin/kafka-server-start.sh config/server.properties &
cd /usr/local/soft/kafka_2.11-2.3.0/
kafka-topics.sh --create --zookeeper hadoop100:2181 --replication-factor 1 --partitions 1 --topic app-crawler
kafka-topics.sh -list -zookeeper hadoop100:2181
kafka-console-consumer.sh --bootstrap-server hadoop100:9092 --topic app-crawler --from-beginning

cd /usr/local/soft/apache-flume-1.9.0-bin/conf/
flume-ng agent -n a1 -c conf -f flume-taildir-kafka.properties -Dflume.root.logger=DEBUG,console

2、kafka-hdfs

创建flume_kafka-hdfs.properties

a1.sources=r1
a1.sinks=k1
a1.channels=c1
​
a1.sources.r1.type=org.apache.flume.source.kafka.KafkaSource
a1.sources.r1.kafka.bootstrap.servers=hadoop100:9092
a1.sources.r1.kafka.topics=app-crawler
​
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://hadoop100:9000/flume
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 10
a1.sinks.k1.hdfs.rollSize = 0
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.filePrefix = %Y-%m-%d-%H-%M-%S
a1.sinks.k1.hdfs.useLocalTimeStamp = true
​
​
a1.channels.c1.type=memory
a1.channels.c1.capacity = 15000
a1.channels.c1.transactionCapacity = 15000
​
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1
a1.sinks.k1.channel=c1

启动

start-all.sh
hadoop fs -mkdir /flume
kafka-console-producer.sh --broker-list hadoop100:9092 --topic app-crawler

cd /usr/local/soft/apache-flume-1.9.0-bin/conf/
flume-ng agent -n a1 -c conf -f flume_kafka-hdfs.properties -Dflume.root.logger=DEBUG,console