Kafka笔记

文修森啦

已于 2023-09-04 22:48:18 修改

阅读量51

点赞数

文章标签： kafka

于 2023-08-02 06:42:41 首次发布

本文链接：https://blog.csdn.net/qq_34216033/article/details/131999035

版权

基础

单机

work path /usr/local/software/kafka

配置

broker.id=0
listeners=PLAINTEXT://wxsl.com:9092
log.dirs=/tmp/kafka-logs
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-standalone

启动

bin/kafka-server-start.sh config/server.properties

验证


bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-standalone --create --topic standalone-verify-topic --partitions 8 --replication-factor 1

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-standalone --describe --topic standalone-verify-topic

集群

配置

node01

mkdir -p cluster/kafka01/data
mkdir -p cluster/kafka01/config

kafka01_cfg="

broker.id=1
port=9092
listeners=PLAINTEXT://wxsl.com:9092
log.dirs=/usr/local/software/kafka/cluster/kafka01/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster

" 
echo $kafka01_cfg > cluster/kafka01/config/server.properties

node02

mkdir -p cluster/kafka02/data
mkdir -p cluster/kafka02/config

kafka02_cfg="

broker.id=2
port=9093
listeners=PLAINTEXT://wxsl.com:9093
log.dirs=/usr/local/software/kafka/cluster/kafka02/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster

" 
echo $kafka02_cfg > cluster/kafka02/config/server.properties

node03

mkdir -p cluster/kafka03/data
mkdir -p cluster/kafka03/config

kafka03_cfg="

broker.id=3
port=9094
listeners=PLAINTEXT://wxsl.com:9094
log.dirs=/usr/local/software/kafka/cluster/kafka03/data
zookeeper.connect=wxsl.com:2181,wxsl.com:2182,wxsl.com:2183/kafka-cluster

" 
echo $kafka03_cfg > cluster/kafka03/config/server.properties

启动

bin/kafka-server-start.sh cluster/kafka01/config/server.properties&
bin/kafka-server-start.sh cluster/kafka02/config/server.properties&
bin/kafka-server-start.sh cluster/kafka03/config/server.properties&

验证



bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic --partitions 8 --replication-factor 2

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --describe --topic cluster-verify-topic

工具

主题

kafka-topics.sh

创建

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic --partitions 8 --replication-factor 2

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --create --topic cluster-verify-topic-config --partitions 8 --replication-factor 2 --config cleanup.policy=compact max.message=bytes=1024

删除

服务端配置delete.topic.enable参数配置要求是true，默认true

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --delete --topic cluster-verify-topic-config

查看

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --list

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --describe --topic cluster-verify-topic

修改

bin/kafka-topics.sh --zookeeper wxsl.com:2181/kafka-cluster --alter --topic cluster-verify-topic --partitions 12

配置

kafka-configs.sh

修改

bin/kafka-configs.sh --zookeeper wxsl.com:2181/kafka-cluster --alter --entity-type topics --entity-name cluster-verify-topic --add-config cleanup.policy=compact,max.message.bytes=1024

查看

bin/kafka-configs.sh --bootstrap-server wxsl.com:9092 --describe --entity-type topics --entity-name cluster-verify-topic

消费

bin/kafka-console-consumer.sh --bootstrap-server wxsl.com:9092 --topic rosalind-test-topic --from-beginning

生产

bin/kafka-console-producer.sh --bootstrap-server wxsl.com:9092 --topic rosalind-test-topic

优先副本

kafka-preferred-replica-election.sh

auto.leader.rebalance.enable 默认值true，关键时期执行时间无法掌控，建议生产环境设置成false

bin/kafka-preferred-replica-election.sh --zookeeper wxsl.com:2181/kafka-cluster

election_json='
{
	"partitions":[
		{
			"partition":0,
			"topic":"cluster-verify-topic"
		},
		{
			"partition":1,
			"topic":"cluster-verify-topic"
		},
		{
			"partition":2,
			"topic":"cluster-verify-topic"
		}
	]
}
'
echo $election_json > election.json

生产环境推荐使用--path-to-json-file 方式分批执行

 bin/kafka-leader-election.sh --bootstrap-server wxsl.com:9092 --election-type preferred  --path-to-json-file election.json

重分区

reassign_json='
{
	"topics":[
		{
			"topic":"cluster-verify-topic"
		}
	]
}
'
echo $reassign_json > reassign.json

生成重分区方案

bin/kafka-reassign-partitions.sh --bootstrap-server wxsl.com:9092 --generate --topics-to-move-json-file reassign.json --broker-list 0,1,2

将生成的重分区方案保存到 project.json

project_json='
{"version":1,"partitions":[{"topic":"cluster-verify-topic","partition":0,"replicas":[2,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":1,"replicas":[3,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":2,"replicas":[1,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":3,"replicas":[2,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":4,"replicas":[3,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":5,"replicas":[1,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":6,"replicas":[2,3],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":7,"replicas":[3,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":8,"replicas":[1,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":9,"replicas":[2,1],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":10,"replicas":[3,2],"log_dirs":["any","any"]},{"topic":"cluster-verify-topic","partition":11,"replicas":[1,3],"log_dirs":["any","any"]}]}
'

echo $project_json > project.json

执行重分区

bin/kafka-reassign-partitions.sh --bootstrap-server wxsl.com:9092 --execute --reassignment-json-file project.json

复制限流

相关配置 follower.replication.throttled.rate和leader.replication.throttled.rate，单位B/S

消费者组

kafka-consumer-groups.sh

查看

bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --list
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --describe --group rosalind-consumer-group
bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --describe --group rosalind-consumer-group --members --verbose

删除



bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --delete --group rosalind-consumer-group

重置


bin/kafka-consumer-groups.sh --bootstrap-server wxsl.com:9092 --group rosalind-consumer-group --all-topics --reset-offsets --to-earliest --execute