1. Disabling automatically create topic
auto.create.topics.enable=false
2. Add topic
bin/kafka-topics.sh --zookeeper zk_host:port/chroot --create --topic my_topic_name --partitions 10 --replication-factor 2 --configretention.bytes=1073741824 --config retention.ms=2073600000
3. Modify topics
(1) add partition
bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --partitions 20
(2) add(update) config
bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --config retention.bytes=2147483648
(3) remove a config
bin/kafka-topics.sh --zookeeper zk_host:port/chroot --alter --topic my_topic_name --delete-config retention.ms
(4) delete topic
bin/kafka-topics.sh --zookeeper zk_host:port/chroot --delete --topic my_topic_name
delete.topic.enable=true
4. Graceful shutdown
controlled.shutdown.enable=true
5. Balancing leadership
bin/kafka-preferred-replica-election.sh --zookeeper zk_host:port/chroot
auto.leader.rebalance.enable=true
6. Checking consumer position
bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper zk_host:port/chroot --group groupname
bin/kafka-run-class.sh kafka.tools.ConsumerOffsetChecker --zookeeper zk_host:port/chroot --group groupname --topic topicname
7. Expanding your cluster
(1) Automatically migrating data to new machines
vim topics-to-move.json
{"topics": [{"topic": "yurun"}], "version":1}
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --topics-to-move-json-file topics-to-move.json --broker-list "0,1" --generate
vim expand-cluster-reassignment.json
{"version":1,"partitions":[{"topic":"yurun","partition":8,"replicas":[0,1]},{"topic":"yurun","partition":7,"replicas":[1,0]},{"topic":"yurun","partition":1,"replicas":[1,0]},{"topic":"yurun","partition":0,"replicas":[0,1]},{"topic":"yurun","partition":5,"replicas":[1,0]},{"topic":"yurun","partition":4,"replicas":[0,1]},{"topic":"yurun","partition":2,"replicas":[0,1]},{"topic":"yurun","partition":3,"replicas":[1,0]},{"topic":"yurun","partition":6,"replicas":[0,1]},{"topic":"yurun","partition":9,"replicas":[1,0]}]}
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file expand-cluster-reassignment.json --execute
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file expand-cluster-reassignment.json --verify
(2) Custom partition assignment and migration
vim custom-reassignment.json
{"version":1,"partitions":[{"topic":"yurun","partition":0,"replicas":[2,3]},{"topic":"yurun","partition":1,"replicas":[2,3]}]}
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file custom-reassignment.json --execute
bin/kafka-reassign-partitions.sh --zookeeper localhost:2181 --reassignment-json-file custom-reassignment.json --verify
8. Decommissioning brokers
use 7
9. Increasing replication factor
use 7