基础
单机
work path /usr/local/software/kafka
配置
broker.id=0
listeners=PLAINTEXT://wxsl.com:9092
log.dirs=/tmp/kafka-logs
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-standalone
启动
bin/kafka-server-start.sh config/server.properties
验证
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-standalone --create --topic standalone-verify-topic --partitions 8 --replication-factor 1
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-standalone --describe --topic standalone-verify-topic
集群
配置
node01
mkdir -p cluster/kafka01/data
mkdir -p cluster/kafka01/config
kafka01_cfg="
broker.id=1
port=9092
listeners=PLAINTEXT://wxsl.com:9092
log.dirs=/usr/local/software/kafka/cluster/kafka01/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster
"
echo $kafka01_cfg > cluster/kafka01/config/server.properties
node02
mkdir -p cluster/kafka02/data
mkdir -p cluster/kafka02/config
kafka02_cfg="
broker.id=2
port=9093
listeners=PLAINTEXT://wxsl.com:9093
log.dirs=/usr/local/software/kafka/cluster/kafka02/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster
"
echo $kafka02_cfg > cluster/kafka02/config/server.properties
node03
mkdir -p cluster/kafka03/data
mkdir -p cluster/kafka03/config
kafka03_cfg="
broker.id=3
port=9094
listeners=PLAINTEXT://wxsl.com:9094
log.dirs=/usr/local/software/kafka/cluster/kafka03/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster
"
echo $kafka03_cfg > cluster/kafka03/config/server.properties
启动
bin/kafka-server-start.sh cluster/kafka01/config/server.properties&
bin/kafka-server-start.sh cluster/kafka02/config/server.properties&
bin/kafka-server-start.sh cluster/kafka03/config/server.properties&
验证
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic --partitions 8 --replication-factor 2
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --describe --topic cluster-verify-topic
工具
主题
kafka-topics.sh
创建
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic --partitions 8 --replication-factor 2
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic-config --partitions 8 --replication-factor 2 --config cleanup.policy=compact max.message=bytes=1024
删除
服务端配置delete.topic.enable
参数配置要求是true
,默认true
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --delete --topic cluster-verify-topic-config
查看
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --list
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --describe --topic cluster-verify-topic
修改
bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --alter --topic cluster-verify-topic --partitions 12
配置
kafka-configs.sh
修改
bin/kafka-configs.sh --zookeeper wxsl.com:2181/kafka-cluster --alter --entity-type topics --entity-name cluster-verify-topic --add-config cleanup.policy=compact,max.message.bytes=1024
查看
bin/kafka-configs.sh --bootstrap-server wxsl.com:9092 --describe --entity-type topics --entity-name cluster-verify-topic
消费
bin/kafka-console-consumer.sh --bootstrap-server wxsl.com:9092 --topic rosalind-test-topic --from-beginning
生产
bin/kafka-console-producer.sh --bootstrap-server wxsl.com:9092 --topic rosalind-test-topic
优先副本
kafka-preferred-replica-election.sh
auto.leader.rebalance.enable
默认值true
,关键时期执行时间无法掌控,建议生产环境设置成false
bin/kafka-preferred-replica-election.sh --zookeeper wxsl.com:2181/kafka-cluster
election_json='
{
"partitions":[
{
"partition":0,
"topic":"cluster-verify-topic"
},
{
"partition":1,
"topic":"cluster-verify-topic"
},
{
"partition":2,
"topic":"cluster-verify-topic"
}
]
}
'
echo $election_json > election.json
生产环境 推荐使用--path-to-json-file
方式分批执行
bin/kafka-leader-election.sh --bootstrap-server wxsl.com:9092 --election-type preferred --path-to-json-file election.json
重分区
reassign_json='
{
"topics":[
{
"topic":"cluster-verify-topic"
}
]
}
'
echo $reassign_json > reassign.json
生成重分区方案
bin/kafka-reassign-partitions.sh --bootstrap-server wxsl.com:9092 --generate --topics-to-move-json-file reassign.json --broker-list 0,1,2
将生成的重分区方案保存到 project.json
project_json='
{"version":1,"partitions":[{"topic":"cluster-verify-topic","partition":0,"replicas":[2,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":1,"replicas":[3,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":2,"replicas":[1,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":3,"replicas":[2,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":4,"replicas":[3,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":5,"replicas":[1,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":6,"replicas":[2,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":7,"replicas":[3,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":8,"replicas":[1,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":9,"replicas":[2,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":10,"replicas":[3,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":11,"replicas":[1,3],"log_dirs":["any","any"]}]}
'
echo $project_json > project.json
执行重分区
bin/kafka-reassign-partitions.sh --bootstrap-server wxsl.com:9092 --execute --reassignment-json-file project.json
复制限流
相关配置 follower.replication.throttled.rate
和leader.replication.throttled.rate
,单位B/S
消费者组
kafka-consumer-groups.sh
查看
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --list
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --describe --group rosalind-consumer-group
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --describe --group rosalind-consumer-group --members --verbose
删除
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --delete --group rosalind-consumer-group
重置
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --group rosalind-consumer-group --all-topics --reset-offsets --to-earliest --execute