03.DockerCompose部署Kafka

最新推荐文章于 2024-08-20 13:48:29 发布

Felix_XY

最新推荐文章于 2024-08-20 13:48:29 发布

阅读量206

点赞数 2

分类专栏： DockerCompose kafka 文章标签： kafka docker docker compose

本文链接：https://blog.csdn.net/qq_32060101/article/details/141352009

版权

DockerCompose 同时被 2 个专栏收录

5 篇文章 0 订阅

订阅专栏

kafka

1 篇文章 0 订阅

订阅专栏

参考链接

https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md
https://github.com/apache/kafka/blob/trunk/docker/examples/README.md（官方）
https://hubgw.docker.com/r/bitnami/kafka

镜像获取

# kafka 官方镜像
# https://github.com/apache/kafka/blob/trunk/docker/README.md
docker pull apache/kafka:3.7.0

# 简单的访问工具
# docker pull obsidiandynamics/kafdrop:4.0.1

部署

本次采用KRAFT方式部署

单机部署（使用bridge网络模式）

bridge 需要暴露一个外部IP供客户端访问，注意关注以下参数的值

KAFKA_ADVERTISED_LISTENERS

创建 docker-compose.yml 文件内容如下

version: '3'
services:
  kafka:
    image: apache/kafka:3.7.0
    hostname: kafka
    container_name: kafka
    restart: on-failure:3
    user: '0'
    ports:
      - 9092:9092
    environment:
      - KAFKA_NODE_ID=1
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka:29093
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT_HOST://10.100.30.99:9092,PLAINTEXT://kafka:19092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_LISTENERS=CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    volumes:
      - ./data:/tmp/kraft-combined-logs
      - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

# 部署
docker compose up -d
# 卸载
docker compose down

单机部署（使用host网络模式）

创建 docker-compose.yml 文件内容如下

version: '3'
services:
  kafka:
    image: apache/kafka:3.7.0
    container_name: kafka
    restart: on-failure:3
    user: '0'
    environment:
      - KAFKA_NODE_ID=1
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@10.100.30.99:9093
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.100.30.99:9092
      - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    network_mode: host
    volumes:
      - ./data:/tmp/kraft-combined-logs
      - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

# 部署
docker compose up -d
# 卸载
docker compose down

单机部署集群（使用bridge网络模式）

bridge 需要暴露一个外部IP供客户端访问，注意关注以下参数的值

KAFKA_ADVERTISED_LISTENERS

创建 docker-compose.yml 文件内容如下

version: '3'

networks:
  kafka-network:
    driver: bridge

services:
  kafka-1:
    image: apache/kafka:3.7.0
    container_name: kafka-1
    hostname: kafka-1
    ports:
      - 29092:9092
    environment:
      - KAFKA_NODE_ID=1
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
      - KAFKA_LISTENERS=PLAINTEXT://:19092,CONTROLLER://:9093,PLAINTEXT_HOST://:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-1:19092,PLAINTEXT_HOST://10.100.30.99:29092
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=4L6g3nShT-eMCtK--tarsier
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    networks:
      - kafka-network
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5


  kafka-2:
    image: apache/kafka:3.7.0
    container_name: kafka-2
    hostname: kafka-2
    ports:
      - 39092:9092
    environment:
      - KAFKA_NODE_ID=2
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
      - KAFKA_LISTENERS=PLAINTEXT://:19092,CONTROLLER://:9093,PLAINTEXT_HOST://:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-2:19092,PLAINTEXT_HOST://10.100.30.99:39092
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=4L6g3nShT-eMCtK--tarsier
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    networks:
      - kafka-network
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

  

  kafka-3:
    image: apache/kafka:3.7.0
    container_name: kafka-3
    hostname: kafka-3
    ports:
      - 49092:9092
    environment:
      - KAFKA_NODE_ID=3
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093
      - KAFKA_LISTENERS=PLAINTEXT://:19092,CONTROLLER://:9093,PLAINTEXT_HOST://:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-3:19092,PLAINTEXT_HOST://10.100.30.99:49092
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=4L6g3nShT-eMCtK--tarsier
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    networks:
      - kafka-network
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

# 部署
docker compose up -d
# 卸载
docker compose down

单机部署集群（使用Host网络模式）

Host 在同一台机器会端口冲突，注意关注以下参数的值

KAFKA_ADVERTISED_LISTENERS
KAFKA_LISTENERS
KAFKA_CONTROLLER_QUORUM_VOTERS

创建 docker-compose.yml 文件内容如下

version: '3'

services:
  kafka-1:
    image: apache/kafka:3.7.0
    container_name: kafka-1
    environment:
      - KAFKA_NODE_ID=1
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.100.30.99:9092
      - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@10.100.30.99:9093,2@10.100.30.99:9193,3@10.100.30.99:9293
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    network_mode: host
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5


  kafka-2:
    image: apache/kafka:3.7.0
    container_name: kafka-2
    environment:
      - KAFKA_NODE_ID=2
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.100.30.99:9192
      - KAFKA_LISTENERS=PLAINTEXT://:9192,CONTROLLER://:9193
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@10.100.30.99:9093,2@10.100.30.99:9193,3@10.100.30.99:9293
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    network_mode: host
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

  

  kafka-3:
    image: apache/kafka:3.7.0
    container_name: kafka-3
    environment:
      - KAFKA_NODE_ID=3
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.100.30.99:9292
      - KAFKA_LISTENERS=PLAINTEXT://:9292,CONTROLLER://:9293
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@10.100.30.99:9093,2@10.100.30.99:9193,3@10.100.30.99:9293
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - CLUSTER_ID=abcdefghijklmnopqrstuv
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_LOG_DIRS=/tmp/kraft-combined-logs
      - KAFKA_INITIAL_BROKER_REGISTRATION_TIMEOUT_MS=600000
    network_mode: host
    # volumes:
    #   - ./data:/tmp/kraft-combined-logs
    #   - ./logs:/opt/kafka/logs
    healthcheck:
      test: ["CMD-SHELL", "nc -vz 127.0.0.1 9092 || exit 1"]
      start_period: 20s
      interval: 10s
      timeout: 30s
      retries: 5

# 部署
docker compose up -d
# 卸载
docker compose down

Felix_XY

关注

2
点赞
踩
3

收藏

觉得还不错? 一键收藏
0
评论
03.DockerCompose部署Kafka

转载请注明出处。
复制链接

扫一扫

专栏目录