Kafka常用命令整理

kinit 环境登录

kinit -kt /etc/security/keytabs/kafka.service.keytab kafka/hzadg-mammut-platform2.server.163.org@BDMS.163.COM;

基本操作

./bin/kafka-console-producer.sh --broker-list 10.201.168.136:6667  --topic test1 --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT

bin/kafka-console-consumer.sh --bootstrap-server hzadg-mammut-platform2.server.163.org:6667 --topic test1 --from-beginning --consumer.config=config/consumer.properties --new-consumer --property security.protocol=SASL_PLAINTEXT

kafka在zk中使用的znode:

[zk: hzadg-mammut-platform2.server.163.org:2181(CONNECTED) 13] ls /
[hbase-secure, zk-test, zookeeper]
[zk: hzadg-mammut-platform2.server.163.org:2181(CONNECTED) 14] ls /
[hbase-secure, zk-test, isr_change_notification, kafka-acl, admin, zookeeper, kafka-acl-changes, consumers, config, controller, brokers, controller_epoch]

producer

sudo su kafka -l -c 'sh /usr/ndp/current/kafka-broker/config/kafka-env.sh && /usr/ndp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667  --topic test1 --producer.config /usr/ndp/current/kafka-broker/config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT'


./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667  --topic test --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT

./bin/kafka-console-producer.sh --broker-list localhost:6667 --topic ATLAS_ENTITIES


 --security-protocol PLAINTEXTSASL


./bin/kafka-console-producer.sh --broker-list 127.0.0.1:6667  --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test2

 ./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667  --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test2

consumer

bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181  --from-beginning --consumer.config=config/consumer.properties --property  security.protocol=SASL_PLAINTEXT  --topic test3
bin/kafka-console-consumer.sh  --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667  --property  security.protocol=SASL_PLAINTEXT --topic test3

bin/kafka-console-consumer.sh  --from-beginning --consumer.config config/consumer.properties --new-consumer --bootstrap-server hzadg-mammut-platform2.server.163.org:6667  --property  security.protocol=SASL_PLAINTEXT --zookeeper hzadg-mammut-platform2.server.163.org:2181  --topic test3

 ./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform2.server.163.org:6667  --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic test3

hzadg-mammut-platform2.server.163.org:2181,hzadg-mammut-platform3.server.163.org:2181,hzadg-mammut-platform4.server.163.org:2181

bin/kafka-console-consumer.sh  --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667 --property  security.protocol=SASL_PLAINTEXT --topic test3

# bin/kafka-console-consumer.sh  --from-beginning --consumer.config=config/consumer.properties --new-consumer --bootstrap-server=hzadg-mammut-platform2.server.163.org:6667,hzadg-mammut-platform2.server.163.org:6668,hzadg-mammut-platform2.server.163.org:6669  --topic test2

创建topic

 ./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181   --topic ATLAS_ENTITIES  --create --partitions 1 --replication-factor  1
  ./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181   --topic test8  --create --partitions 1 --replication-factor  1

list topic

bin/kafka-topics.sh –zookeeper hzadg-mammut-platform2.server.163.org:2181 –list

topic状态

./bin/kafka-topics.sh –describe –zookeeper hzadg-mammut-platform2.server.163.org:2181 –topic test6

删除topic

bin/kafka-topics.sh –zookeeper hzadg-mammut-platform2.server.163.org:2181 –delete –topic ATLAS_HOOK

授权list

bin/kafka-acls.sh --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --group atlas --list --topic ATLAS_HOOK

./bin/kafka-acls.sh  --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181  --add  --topic test8  --allow-principal User:* --producer

./bin/kafka-acls.sh  --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add  --topic test7 --allow-principal User:* --consumer --group test-consumer-group

启动kafka

source /usr/ndp/current/kafka-broker/config/kafka-env.sh ; /usr/ndp/current/kafka-broker/bin/kafka.sh start

bin/kafka-console-producer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181  --topic ATLAS_HOOK --property "security-protocol=PLAINTEXTSASL"


bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform1.server.163.org:6667  --topic ATLAS_HOOK

bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_HOOK  --from-beginning  --property "security-protocol=SASL_PLAINTEXT"


bin/kafka-console-consumer.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181 --topic ATLAS_HOOK --security-protocol PLAINTEXTSASL --from-beginning


advertised.host.name

---

其他命令参考：

rmr /brokers/topics/__consumer_offsets

  ./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181   --topic test8  --create --partitions 1 --replication-factor  1

./bin/kafka-acls.sh  --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181  --add  --topic test8  --allow-principal User:* --producer --group test-consumer-group

./bin/kafka-acls.sh  --authorizer-properties zookeeper.connect=hzadg-mammut-platform2.server.163.org:2181 --add  --topic ATLAS_HOOK --allow-principal User:* --consumer --group test-consumer-group


./bin/kafka-topics.sh --zookeeper hzadg-mammut-platform2.server.163.org:2181   --topic test4  --create --partitions 1 --replication-factor  1

./bin/kafka-console-producer.sh --broker-list hzadg-mammut-platform1.server.163.org:6667  --producer.config config/producer.properties --producer-property security.protocol=SASL_PLAINTEXT --topic ATLAS_HOOK


 bin/kafka-console-consumer.sh  --from-beginning --consumer.config config/consumer.properties --new-consumer --bootstrap-server hzadg-mammut-platform2.server.163.org:6667 --property security.protocol=SASL_PLAINTEXT   --topic ATLAS_HOOK

参考：
* http://www.infoq.com/cn/articles/kafka-analysis-part-7
* http://blog.csdn.net/u012373815/article/details/53648757
* https://docs.confluent.io/3.1.1/streams/architecture.html#streams-architecture
* https://gist.github.com/timothyrenner/a99c86b2d6ed2c22c8703e8c7760af3a
* https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51809888
* https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Improvement+Proposals
* confluent Authorization and ACLs: https://docs.confluent.io/current/kafka/authorization.html
* https://cwiki.apache.org/confluence/display/KAFKA/KIP-11+-+Authorization+Interface
* IBM Kafka ACLs in Practice: https://developer.ibm.com/opentech/2017/05/31/kafka-acls-in-practice/
* Kafka ACL: http://orchome.com/185
* kafka 官网： https://kafka.apache.org/documentation/