kafka脚本操作

版本：2.11-0.10.1.0

----------------------------------------------------------------------------------------------------------------------------

kafka-server-start.sh-----启动工具

后台运行指定配置文件

./bin/kafka-server-start.sh -daemon ./config/server.properties

----------------------------------------------------------------------------------------------------------------------------

kafka-server-stop.sh-----停止工具

./bin/kafka-server-stop.sh 停止kafka运行

----------------------------------------------------------------------------------------------------------------------------

kafka-topics.sh-----topic-----topic管理工具

./bin/kafka-topics.sh --list --zookeeper localhost:2181 查看有哪些topic

./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic yaocl-test 创建yaocl-test 主题，一个副本一个分区

./bin/kafka-topics.sh --topic yaocl-test --describe --zookeeper localhost:2181 查看yaocl-test主题的详细信息

./bin/kafka-topics.sh --delete --zookeeper group1-master-2:2182 --topic test1 删除test1 topic

./bin/kafka-topics.sh --zookeeper group1-master-2:2182 --alter --topic test2 --partitions 4 将test2 topic分区增加到4个

----------------------------------------------------------------------------------------------------------------------------

kafka-configs.sh-----kafka在线配置工具

./bin/kafka-configs.sh --zookeeper localhost:2181 --entity-type topics --entity-name yaocl-test --alter --add-config retention.ms=43200000 kafka修改yaocl-test主题日志保留时间参数

----------------------------------------------------------------------------------------------------------------------------

kafka-acls.sh-----kafka ACL认证工具

./bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --operation Write --operation Read --allow-principal User:* --allow-host 192.168.0.175 --add --topic test 允许所有用户从192.168.0.175对test topic可写，可读

./kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --list --topic test 查看test主题上的acl策略有哪些

./kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:2181 --operation Write --allow-principal User:* --allow-host 192.168.0.175 --remove --topic yaocl-test 只删除可写的策略

----------------------------------------------------------------------------------------------------------------------------

kafka-consumer-groups.sh-----消费组工具

./bin/kafka-consumer-groups.sh --new-consumer --bootstrap-server localhost:9092 --list 查看有哪些消费组

注：也可以用api接口查看

./bin/kafka-consumer-groups.sh --new-consumer --bootstrap-server group1-node-1:9092 --describe -group group-to-sy-02 查看消费组group-to-sy-02情况

----------------------------------------------------------------------------------------------------------------------------

kafka-run-class.sh-----kafka偏移量查询工具

./bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic veh_diagnose_org_gw --time -1 --broker-list group1-node-1:9092 --partitions 1 查看topic某分区偏移量最大值

./bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic veh_diagnose_org_gw --time -2 --broker-list group1-node-1:9092 --partitions 1 查看topic某分区偏移量最小值

./bin/kafka-run-class.sh kafka.tools.DumpLogSegments --files /mapbar/kafka/test2-3/00000000000000000000.log --print-data-log 验证日志索引是否正确，或者仅仅想从log文件中直接打印消息

----------------------------------------------------------------------------------------------------------------------------

kafka-replay-log-producer.sh-----从一个Kafka集群里面读取指定Topic的消息，并将这些消息发送到其他集群的指定topic中

----------------------------------------------------------------------------------------------------------------------------

kafka-preferred-replica-election.sh-----kafka分区副本工具

./bin/kafka-preferred-replica-election.sh --zookeeper group1-master-2:2182 平衡leader

----------------------------------------------------------------------------------------------------------------------------

kafka-producer-perf-test.sh-----压测工具发送消息

./bin/kafka-producer-perf-test.sh --topic test2 --num-records 100 --record-size 100 --throughput 100 --producer-props bootstrap.servers=group1-node-1:9092

注：

--num-records 发送消息数量

--record-size 发送一个消息的字节数

--throughput 一秒几条记录

----------------------------------------------------------------------------------------------------------------------------

kafka-consumer-perf-test.sh-----压测工具2消费消息

./kafka-consumer-perf-test.sh --zookeeper group1-master-2:2182 --messages 50000000 --topic test2 --threads 1

注：

-messages 消息数量

--threads 消费者使用几个线程同时消费

----------------------------------------------------------------------------------------------------------------------------

kafka-console-consumer.sh-----消费工具

./bin/kafka-console-consumer.sh --bootstrap-server group1-node-2:9092 --topic test2 --new-consumer 实时消费

./bin/kafka-console-consumer.sh --bootstrap-server group1-node-2:9092 --topic test2 --new-consumer --from-beginning 从头消费

./bin/kafka-console-consumer.sh --new-consumer --bootstrap-server 1.202.136.14:9099 --topic veh_diagnose_org_gw --consumer.config test.config ssl链接消费数据

注：test.config配置文件如下

#group.id=test-02

security.protocol=SSL

ssl.truststore.location=/mapbar/app/kafka/ssl/client.truststore.jks

ssl.truststore.password=mapbar

ssl.keystore.location=/mapbar/app/kafka/ssl/client.keystore.jks

ssl.keystore.password=mapbar

ssl.key.password=mapbar

----------------------------------------------------------------------------------------------------------------------------

kafka-console-producer.sh-----生产数据工具

./bin/kafka-console-producer.sh --broker-list 192.168.15.249:9092 --topic posraw 生产数据

./bin/kafka-console-producer.sh --broker-list 1.202.136.14:9099 --topic veh_diagnose_org_gw --producer.config test.config ssl链接生产数据

注：test.config配置文件如下

#group.id=test-02

security.protocol=SSL

ssl.truststore.location=/mapbar/app/kafka/ssl/client.truststore.jks

ssl.truststore.password=mapbar

ssl.keystore.location=/mapbar/app/kafka/ssl/client.keystore.jks

ssl.keystore.password=mapbar

ssl.key.password=mapbar

----------------------------------------------------------------------------------------------------------------------------

kafka-reassign-partitions.sh-----副本操作工具

./bin/kafka-reassign-partitions.sh --zookeeper group1-master-2:2182,group1-master-1:2182,group1-client:2812 --reassignment-json-file replication.json --execute 增加副本，需要自己写json文件，如下：

{"version":1,"partitions":[{"topic":"test2","partition":0,"replicas":[3,2,1]},{"topic":"test2","partition":1,"replicas":[1,3,2]},{"topic":"test2","partition": 2,"replicas":[2,1,3]},{"topic":"test2","partition":3,"replicas":[3,2,1]}]}