Flume + Kafka 结合案例

Flume + Kafka 结合案例

创建Kafka主题

启动kafka服务

kafka-server-start.sh /opt/software/kafka/kafka280scala212/config/kraft/server.properties

创建主题

kafka-topics.sh --create --topic prolog_01 --partitions 1 --replication-factor 1 --bootstrap-server singlechen:9092

创建Flume配置文件

vim /root/flume_job/conf/flume07.conf

#组件声明
a1.sources = s1
a1.channels = c1
a1.sinks = k1
#初始化数据源
a1.sources.s1.type=taildir
a1.sources.s1.filegroups=f1  #可以有多个文件源
a1.sources.s1.filegroups.f1=/root/data/flume/log01/prolog.log.bak #可以针对多个文件
a1.sources.s1.positionFile=/root/data/flume/taildir02/taildir_prolog.json  #记录读取位置的路径
a1.sources.s1.batchSize =10
#初始化通道
a1.channels.c1.type=file
a1.channels.c1.checkpointDir=/opt/software/flume/flume190/mydata/checkpoint02
a1.channels.c1.dataDirs=/opt/software/flume/flume190/mydata/data02
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100  #transactionCapacity >= 100
#初始化数据槽
a1.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.bootstrap.servers = singlechen:9092
a1.sinks.k1.kafka.topic = prolog_01
a1.sinks.k1.kafka.flumeBatchSize = 10
a1.sinks.k1.kafka.producer.linger.ms = 500
#关联组件
a1.sources.s1.channels=c1
a1.sinks.k1.channel=c1
4.启动kafka控制台消费者

启动kafka控制台消费者

kafka-console-consumer.sh --bootstrap-server singlechen:9092 --topic prolog_01 --from-beginning

执行Flume

flume-ng agent -name a1 -c /opt/software/flume/flume190/conf/ -f /root/flume_job/conf/flume07.conf -Dflume.root.logger=INFO,console

更多应用场景示意