kafka配置和使用

官方文档

http://kafka.apache.org/

下载安装

wget http://mirror.bit.edu.cn/apache/kafka/1.0.0/kafka_2.11-1.0.0.tgz -P /opt/
cd  /opt
tar -xzf kafka_2.11-1.0.0.tgz
cd kafka_2.11-1.0.0

配置

启动

bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
bin/kafka-server-start.sh -daemon config/server.properties

停止

kafka-server-stop.sh

消费者

bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.144:9092,192.168.1.145:9092,192.168.1.146:9092 --topic topic1 --from-beginning --max-messages 10

生产者

bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
This is a message
This is another message

其他

查看topic的详细信息

./kafka-topics.sh -zookeeper 127.0.0.1:2181 -describe -topic testKJ1

查看所有topic

./kafka-topics.sh --list --zookeeper 127.0.0.1:2181

为topic增加副本

./kafka-reassign-partitions.sh -zookeeper 127.0.0.1:2181 -reassignment-json-file json/partitions-to-move.json -execute

创建topic

bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic testKJ1

为topic增加partition.

./bin/kafka-topics.sh  --alter --zookeeper 127.0.0.1:2181 --partitions 2 --topic testKJ1

kafka生产者客户端命令

./kafka-console-producer.sh --broker-list localhost:9092 --topic testKJ1

kafka消费者客户端命令

./kafka-console-consumer.sh -zookeeper localhost:2181 --from-beginning --topic testKJ1

kafka服务启动

./kafka-server-start.sh -daemon ../config/server.properties

###下线broker
./kafka-run-class.sh kafka.admin.ShutdownBroker --zookeeper 127.0.0.1:2181 --broker #brokerId# --num.retries 3 --retry.interval.ms 60shutdown broker

删除topic

./kafka-run-class.sh kafka.admin.DeleteTopicCommand --topic testKJ1 --zookeeper 127.0.0.1:2181
./kafka-topics.sh --zookeeper localhost:2181 --delete --topic testKJ1

查看consumer组内消费的offset

./kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper localhost:2181 --group test --topic testKJ1
bin/kafka-consumer-groups.sh --bootstrap-server  localhost:6667  --describe  --group groupName

最大字节调整

生产端
max.request.size=10485760
消费端
fetch.message.max.bytes=xxx

message.max.bytes=10000000
replica.fetch.max.bytes=10485760

kafka 查询所有分区数

sh kafka-topics.sh --describe --zookeeper localhost:2181 |grep -v “PartitionCount” |wc -l

开发
–生产者
bin/kafka-console-producer.sh --broker-list localhost:6667 --topic topicName

bin/kafka-topics.sh -zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 -describe -topic topicName
bin/kafka-topics.sh --list --zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181
bin/kafka-topics.sh --alter --zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --partitions 10 --topic topicName
bin/kafka-topics.sh --zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --delete --topic topicName
bin/kafka-topics.sh --create --zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --replication-factor 2 --partitions 3 --topic topicName
bin/kafka-console-consumer.sh -zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --topic topicName
bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --group group1 --topic topicName

bin/kafka-console-consumer.sh -zookeeper host02:2181,host01:2181,host03:2181,host04:2181,host05:2181 --topic topicName

–测试
–查看group.id 消费的offset
bin/kafka-consumer-groups.sh --bootstrap-server host02:9092 --describe --group groupName
bin/kafka-console-consumer.sh -zookeeper host01:2181,host02:2181,host03:2181 --topic topicName
bin/kafka-console-consumer.sh -zookeeper host01:2181,host02:2181,host03:2181 --topic topicName
bin/kafka-topics.sh --list --zookeeper host01:2181,host02:2181,host03:2181