一、kakfa生产与消费:验证zookeeper和kafka状态
### --- 验证zookeeper状态
[root@hadoop ~]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/yanqi/servers/zookeeper/bin/../conf/zoo.cfg
Mode: standalone
### --- 启动kafka
[root@hadoop ~]# kafka-server-start.sh -daemon /opt/yanqi/servers/kafka/config/server.properties
二、生产与消费
### --- kafka-topics.sh 用于管理主题。
~~~ # 列出现有的主题
[root@hadoop ~]# kafka-topics.sh --list --zookeeper localhost:2181/myKafka
~~~ # 创建主题,该主题包含一个分区,该分区为Leader分区,它没有Follower分区副本。
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka \
--create --topic topic_1 --partitions 1 --replication-factor 1
~~~ # 查看分区信息
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka --list
~~~ 输出参数
topic_1
~~~ # 查看指定主题的详细信息
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka \
--describe --topic topic_1
~~~ 输出参数
Topic:topic_1 PartitionCount:1 ReplicationFactor:1 Configs:
Topic: topic_1 Partition: 0 Leader: 0 Replicas: 0 Isr: 0
~~~ # 注
~~~ 当前的主题是:topic_1
~~~ 当前主题有几个分区:PartitionCount:1
~~~ 副本因子是:ReplicationFactor:1
~~~ # 删除指定主题
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka \
--delete --topic topic_1
~~~ 输出参数
Topic topic_1 is marked for deletion.
Note: This will have no impact if delete.topic.enable is not set to true.
~~~ # 创建topic_2主题,有1个副本5个分区
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka \
--create --topic topic_2 --partitions 5 --replication-factor 1
~~~ # 查看主题
[root@hadoop ~]# kafka-topics.sh --zookeeper localhost:2181/myKafka \
--describe --topic topic_2
Topic:topic_2 PartitionCount:5 ReplicationFactor:1 Configs:
Topic: topic_2 Partition: 0 Leader: 0 Replicas: 0 Isr: 0
Topic: topic_2 Partition: 1 Leader: 0 Replicas: 0 Isr: 0
Topic: topic_2 Partition: 2 Leader: 0 Replicas: 0 Isr: 0
Topic: topic_2 Partition: 3 Leader: 0 Replicas: 0 Isr: 0
Topic: topic_2 Partition: 4 Leader: 0 Replicas: 0 Isr: 0
### --- kafka-console-producer.sh用于生产消息:
~~~ # 开启生产者
[root@hadoop ~]# kafka-console-producer.sh --topic topic_1 \
--broker-list localhost:9092
~~写入参数
>yanqi——1
>yanqi——2
>yanqi——3
### --- kafka-console-consumer.sh用于消费消息:
~~~ # 开启消费者
[root@hadoop ~]# kafka-console-consumer.sh --bootstrap-server localhost:9092 \
--topic topic_1
~~~ 输出参数
yanqi——1
yanqi——2
yanqi——3
~~~ # 开启消费者方式二,从头消费,不按照偏移量消费
[root@hadoop ~]# kafka-console-consumer.sh --bootstrap-server localhost:9092 \
--topic topic_1 --from-beginning
~~~ 输出参数
hello world
hello world
yanqi——1
yanqi——2
yanqi——3
### --- 在磁盘上查看生成消费的数据
~~~ # 该目录下打开了很多偏移量
~~~ 这些都是kafka生成消费的数据,包括主题分区详情。
[root@hadoop ~]# ls /opt/yanqi/servers/kafka/kafka-logs/
cleaner-offset-checkpoint __consumer_offsets-15 __consumer_offsets-22 __consumer_offsets-3 __consumer_offsets-37 __consumer_offsets-44 __consumer_offsets-7 topic_2-0
__consumer_offsets-0 __consumer_offsets-16 __consumer_offsets-23 __consumer_offsets-30 __consumer_offsets-38 __consumer_offsets-45 __consumer_offsets-8 topic_2-1
__consumer_offsets-1 __consumer_offsets-17 __consumer_offsets-24 __consumer_offsets-31 __consumer_offsets-39 __consumer_offsets-46 __consumer_offsets-9 topic_2-2
__consumer_offsets-10 __consumer_offsets-18 __consumer_offsets-25 __consumer_offsets-32 __consumer_offsets-4 __consumer_offsets-47 log-start-offset-checkpoint topic_2-3
__consumer_offsets-11 __consumer_offsets-19 __consumer_offsets-26 __consumer_offsets-33 __consumer_offsets-40 __consumer_offsets-48 meta.properties topic_2-4
__consumer_offsets-12 __consumer_offsets-2 __consumer_offsets-27 __consumer_offsets-34 __consumer_offsets-41 __consumer_offsets-49 recovery-point-offset-checkpoint
__consumer_offsets-13 __consumer_offsets-20 __consumer_offsets-28 __consumer_offsets-35 __consumer_offsets-42 __consumer_offsets-5 replication-offset-checkpoint
__consumer_offsets-14 __consumer_offsets-21 __consumer_offsets-29 __consumer_offsets-36 __consumer_offsets-43 __consumer_offsets-6 topic_1-0