启动flume
在安装目录下执行
flume-ng.cmd agent -conf ../conf -conf-file ../conf/example.conf -name a1 -property flume.root.logger=INFO,console
启动zookeeper
.\bin\windows\zookeeper-server-start.bat .\config\zookeeper.properties
启动kafka
在安装目录下
.\bin\windows\kafka-server-start.bat .\config\server.properties
创建topic测试主题
在kafka安装路径下
.\bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
创建生产者产生消息
在kafka安装路径下
.\bin\windows\kafka-console-producer.bat --broker-list localhost:9092 --topic test
创建消费者接收消息
在kafka安装路径下
.\bin\windows\kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic test --from-beginning
修改example.conf中的文件
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.bind = 192.168.0.12
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = logger
# 配置Sink组件,将接收数据输出到对应kafka
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = 127.0.0.1:9092
a1.sinks.k1.kafka.topic = test
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
a1.sinks.k1.kafka.producer.compression.type = snappy
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
a1.sinks.k1.type = logger,必须在配置Sink组件上
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
# 本地ip,端口号
a1.sinks.k1.kafka.bootstrap.servers = 127.0.0.1:9092
#kafka主题交换机名称
a1.sinks.k1.kafka.topic = test
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
a1.sinks.k1.kafka.producer.compression.type = snappy
springboot项目
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
启动springboot项目
在flume控制台如下结果
在消费者控制台