单机模式 docker-compose 文件 kafka.yml
- 访问端口是EXTERNAL://:9094, 需要配置/etc/hosts 域名映射
version: "3"
services:
kafka:
image: 'bitnami/kafka:3.4'
hostname: 'kafka'
container_name: kafka
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_CFG_NODE_ID=1
- NODE_ID_COMMAND=1
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_BROKER_ID=1
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092,EXTERNAL://kafka:9094
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- ./data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
kafka-ui:
container_name: kafka-ui
image: provectuslabs/kafka-ui:latest
ports:
- 8080:8080
depends_on:
- "kafka"
environment:
KAFKA_CLUSTERS_0_NAME: local
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9094
AUTH_TYPE: "LOGIN_FORM"
SPRING_SECURITY_USER_NAME: admin
SPRING_SECURITY_USER_PASSWORD: admin
volumes:
- /etc/hosts:/etc/hosts
集群模式五台机器
version: "3"
services:
kafka-master:
image: 'bitnami/kafka:3.4'
hostname: 'kafka1'
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka1
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_BROKER_ID=1
- KAFKA_CFG_NUM_PARTITIONS=3
- KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://:9092,EXTERNAL://kafka1:9094
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093,2@kafka2:9093, \
3@kafka3:9093,4@kafka4:9093,5@kafka5:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- /data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
kafka-ui:
container_name: kafka-ui
image: provectuslabs/kafka-ui:latest
ports:
- 8080:8080
depends_on:
- "kafka-master"
environment:
KAFKA_CLUSTERS_0_NAME: local
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka1:9094,kafka2:9094, \
kafka3:9094,kafka4:9094,kafka5:9094
AUTH_TYPE: "LOGIN_FORM"
SPRING_SECURITY_USER_NAME: admin
SPRING_SECURITY_USER_PASSWORD: admin
volumes:
- /etc/hosts:/etc/hosts
version: "3"
services:
kafka-broker1:
image: 'bitnami/kafka:3.4'
hostname: 'kafka2'
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka2
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_BROKER_ID=2
- KAFKA_CFG_NUM_PARTITIONS=3
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://:9092,EXTERNAL://kafka2:9094
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093,2@kafka2:9093, \
3@kafka3:9093,4@kafka4:9093,5@kafka5:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- /data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
version: "3"
services:
kafka-broker2:
image: 'bitnami/kafka:3.4'
hostname: 'kafka3'
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka3
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_BROKER_ID=3
- KAFKA_CFG_NUM_PARTITIONS=3
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://:9092,EXTERNAL://kafka3:9094
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093,2@kafka2:9093, \
3@kafka3:9093,4@kafka4:9093,5@kafka5:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- /data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
version: "3"
services:
kafka-broker3:
image: 'bitnami/kafka:3.4'
hostname: 'kafka4'
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka4
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_BROKER_ID=4
- KAFKA_CFG_NUM_PARTITIONS=3
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://:9092,EXTERNAL://kafka4:9094
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093,2@kafka2:9093, \
3@kafka3:9093,4@kafka4:9093,5@kafka5:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- /data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
version: "3"
services:
kafka-broker4:
image: 'bitnami/kafka:3.4'
hostname: 'kafka5'
user: root
ports:
- '9092:9092'
- '9093:9093'
- '9094:9094'
environment:
- KAFKA_ADVERTISED_HOST_NAME=kafka5
- KAFKA_ENABLE_KRAFT=yes
- KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
- KAFKA_CFG_PROCESS_ROLES=broker,controller
- KAFKA_BROKER_ID=5
- KAFKA_CFG_NUM_PARTITIONS=3
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
- KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094,CONTROLLER://:9093
- KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://:9092,EXTERNAL://kafka5:9094
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093,2@kafka2:9093, \
3@kafka3:9093,4@kafka4:9093,5@kafka5:9093
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
- KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=240000
volumes:
- /data/quake_kafka/kafka:/bitnami/kafka/
- /etc/hosts:/etc/hosts
注意事项
- topic __consumer_offsets 需要多个副本,否则一个副本那台机器挂了,kafka集群就会停摆
- kafka hostname映射ip 每台机器都需要同步, 包括消费者和生产者机器
kafka1 x.x.x.x
kafka2 x.x.x.x
kafka3 x.x.x.x
kafka4 x.x.x.x
kafka5 x.x.x.x
- KAFKA_CFG_NUM_PARTITIONS=3和KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3 参数不能大于集群台数
- 参数解释
这些参数是用于配置和设置 Apache Kafka 的各种属性和功能。下面是对每个参数的解释:
KAFKA_ADVERTISED_HOST_NAME: 这是 Kafka 服务器的主机名或 IP
地址,用于通知生产者和消费者连接到该地址。
KAFKA_ENABLE_KRAFT: 这个参数启用 Kafka Raft(KRaft)模式,它是 Kafka 2.8.0
版本引入的一种新的高可用性模式。
KAFKA_KRAFT_CLUSTER_ID: 这是 KRaft 集群的唯一标识符,用于将 Kafka 服务器加入特定的 KRaft
集群。
KAFKA_CFG_PROCESS_ROLES: 这个参数指定 Kafka 服务器的角色,其中 "broker" 表示服务器是一个普通的
Kafka 节点,"controller" 表示服务器是集群的控制器。
KAFKA_BROKER_ID: 这是 Kafka 服务器的唯一标识符,用于在集群中区分不同的服务器。
KAFKA_CFG_NUM_PARTITIONS: 这个参数指定每个主题的默认分区数。分区是 Kafka
中的消息存储单元,用于实现消息的并行处理。
KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR:
这个参数指定偏移量主题的副本因子,偏移量主题用于存储消费者的偏移量信息。
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP:
这个参数定义了不同监听器的安全协议映射关系。在这个例子中,"CLIENT"、"EXTERNAL" 和 "CONTROLLER"
分别映射到 "PLAINTEXT" 协议,表示这些监听器使用明文传输。
KAFKA_CFG_LISTENERS: 这个参数定义了 Kafka
监听器的配置。在这个例子中,有三个监听器分别使用不同的协议和端口号。
KAFKA_CFG_ADVERTISED_LISTENERS: 这个参数指定了广告监听器的配置,用于告知生产者和消费者连接到 Kafka
的地址和端口号。
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 这个参数指定了经纪人之间通信使用的监听器名称,其中
"CLIENT" 表示使用客户端监听器。
KAFKA_CFG_CONTROLLER_LISTENER_NAMES:
这个参数指定控制器监听器的名称,用于控制器与其他经纪人之间的通信。
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 这个参数指定了控制器选举过程中的投票者信息,包括其 ID
和监听器的地址。
ALLOW_PLAINTEXT_LISTENER: 这个参数允许使用明文传输的监听器进行连接。
KAFKA_AUTO_CREATE_TOPICS_ENABLE:
这个参数启用自动创建主题的功能,当生产者发送消息到不存在的主题时,将自动创建该主题。
KAFKA_CFG_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS:
这个参数指定初始经纪人注册的超时时间,以毫秒为单位。在集群启动期间,经纪人必须在指定的时间内完成注册。
这些参数可以根据实际需求进行配置和调整,以满足特定的 Kafka 集群设置和功能要求。