Flume与Kafka整合案例详解

环境配置

名称	版本	下载地址
Centos 7.0	64x	百度
Zookeeper	3.4.5
Flume	1.6.0
Kafka	2.1.0

配置Flume

这里就不介绍了零基础出门右转看Flume的文章

flume笔记

直接贴配置文件

[root@zero239 kafka_2.10-0.10.1.1]# cat /opt/hadoop/apache-flume-1.6.0-bin/conf/kafka-conf.properties 
# The configuration file needs to define the sources,
# the channels and the sinks.
# Sources, channels and sinks are defined per agent,
# in this case called 'agent'

agent.sources = r1
agent.channels = c1
agent.sinks = s1

# For each one of the sources, the type is defined
#agent.sources.r1.type = spooldir
#agent.sources.r1.command = /opt/test/logs/data
#agent.sources.r1.fileHeader = true
#agent.sources.r1.channels = c1

agent.sources.r1.type = spooldir
agent.sources.r1.spoolDir = /opt/test/logs/data
agent.sources.r1.fileHeader = true

# Each sink's type must be defined

#agent.sinks.s1.type = logger

agent.sinks.s1.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.s1.topic = logstest
agent.sinks.s1.brokerList = zero230:9092
agent.sinks.s1.requiredAcks = 1
agent.sinks.s1.batchSize = 2


# Each channel's type is defined.
agent.channels.c1.type = memory
agent.channels.c1.capacity = 100
agent.sources.r1.channels = c1
agent.sinks.s1.channel = c1

配置Kafka

# Licensed to the Apache Software Foundation (ASF) under one or more
# con