1, 使用版本如下:
flume-1.7.0
kafka-2.11.0
zookeeper-3. 4.9
2, 配置flume, 源数据基于 日志文件内容检测, sink为 kafka 的producer, 配置文件如下:
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.channels = c1
a1.sources.r1.command = tail -F /root/test.log
# Describe the sink
a1.sinks.k1.type= org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.brokerList=MASTER:9092
a1.sinks.k1.topic=testKafka
a1.sinks.k1.serializer.class=kafka.serializer.StringEncoder
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
3, 为kafka创建topic --> testKafka
kafka-topics.sh -zookeeper localhost:2181 -describe -topic testKafka
4, 先运行zookeeper 和kafka
[root@MASTER conf]# zkServer.sh start
[root@MASTER conf]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /root/cluster/opt/zookeeper-3.4.9/bin/../conf/zoo.cfg
Mode: standalone
[root@MASTER conf]# kafka-server-start.sh -daemon ../conf/server.properties
查看进程:
[root@MASTER conf]# jps
13010 SecondaryNameNode
27651 Kafka
12534 NameNode
29207 Jps
23303 QuorumPeerMain
12727 DataNode
24953 ConsoleConsumer
13484 NodeManager
13261 ResourceManager
5, 运行flume
[root@MASTER opt]# flume-ng agent -c . -f kafka.conf -n a1 -Dflume.root.logger=INFO,console
6, 启动kafka 消费者进程
[root@MASTER config]# kafka-console-consumer.sh -zookeeper localhost:2181 --from-beginning --topic testKafka
7, 测试与验证
1),往/root/test.log 写入内容
echo "mytestKafka one >> test.log
echo "mytestKafka two >> test.log ...
2) , 查看kafka消费进程输出:
...
mytestKafka one
mytestKafka two
mytestKafka three ..
echo "mytestKafka one >> test.log
echo "mytestKafka two >> test.log ...
2) , 查看kafka消费进程输出:
...
mytestKafka one
mytestKafka two
mytestKafka three ..