docker 搭建zookeepr集群和kafka集群

搭建kafka集群需要先建好zookeeper集群。我们需要三个镜像，zookeeper、wurstmeister/kafka:2.12-2.0.1和sheepkiller/kafka-manager，其中sheepkiller/kafka-manager不是必须的。

下拉镜像

docker pull zookeeper

docker pull wurstmeister/kafka:2.12-2.0.1

docker pull sheepkiller/kafka-manager

需要用到的服务编排工具 docker compose，下载Docker Compose的当前稳定版本

curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

加权限

chmod +x /usr/local/bin/docker-compose

测试安装是否成功，从下图可以看到docker-compose当前的版本号和权限状态

docker-compose --version

然后创建网络环境

docker network create --subnet=172.19.0.0/24 br17219

docker network create --driver bridge --subnet 172.23.0.0/25 --gateway 172.23.0.1 zookeeper_network 

创建docker-compose.yml文件，内容是：

version: '2'

services:

  zoo1:
    image: zookeeper # 镜像
    restart: always # 重启
    container_name: zoo1
    hostname: zoo1
    ports:
    - "2181:2181"
    volumes:
    - "./zooConfig/zoo.cfg:/conf/zoo.cfg" # 配置
    - "/root/zk1/data:/data"
    - "/root/zk1/datalog:/datalog"
    environment:
      ZOO_MY_ID: 1 # id
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.11

  zoo2:
    image: zookeeper
    restart: always
    container_name: zoo2
    hostname: zoo2
    ports:
    - "2182:2181"
    volumes:
    - "./zooConfig/zoo.cfg:/conf/zoo.cfg"
    - "/root/zk2/data:/data"
    - "/root/zk2/datalog:/datalog"
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.12

  zoo3:
    image: zookeeper
    restart: always
    container_name: zoo3
    hostname: zoo3
    ports:
    - "2183:2181"
    volumes:
    - "./zooConfig/zoo.cfg:/conf/zoo.cfg"
    - "/root/zk3/data:/data"
    - "/root/zk3/datalog:/datalog"
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.13

  kafka1:
    image: wurstmeister/kafka:2.12-2.0.1 # 镜像
    restart: always
    container_name: kafka1
    hostname: kafka1
    ports:
    - 9092:9092
    - 9999:9999
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: PLAINTEXT://172.23.0.14:9092
    volumes:
    - /etc/localtime:/etc/localtime
    - "/root/kafka1/logs:/kafka"
    links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.14

  kafka2:
    image: wurstmeister/kafka:2.12-2.0.1
    restart: always
    container_name: kafka2
    hostname: kafka2
    ports:
    - 9093:9092
    - 9998:9999
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: PLAINTEXT://172.23.0.15:9092
    volumes:
    - /etc/localtime:/etc/localtime
    - "/root/kafka2/logs:/kafka"
    links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.15

  kafka3:
    image: wurstmeister/kafka:2.12-2.0.1
    restart: always
    container_name: kafka3
    hostname: kafka3
    ports:
    - 9094:9092
    - 9997:9999
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_BROKER_ID: 0
      KAFKA_LISTENERS: PLAINTEXT://172.23.0.16:9092
    volumes:
    - /etc/localtime:/etc/localtime
    - "/root/kafka3/logs:/kafka"
    links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      default:
        ipv4_address: 172.23.0.16
kafka-manager:
    image: sheepkiller/kafka-manager
    restart: always
    container_name: kafka-manager
    hostname: kafka-manager
    ports:
    - 9000:9000
    links:
    - kafka1
    - kafka2
    - kafka3
    - zoo1
    - zoo2
    - zoo3
    environment:
      ZK_HOSTS: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_BROKERS: kafka1:9092,kafka2:9092,kafka3:9092
      APPLICATION_SECRET: letmein
      KAFKA_MANAGER_AUTH_ENABLED: "true" # 开启验证
      KAFKA_MANAGER_USERNAME: "admin" # 用户名
      KAFKA_MANAGER_PASSWORD: "admin" # 密码
      KM_ARGS: -Djava.net.preferIPv4Stack=true

networks:
  default:
    external:
      name: zookeeper_network

注意这个路径，我们要现在这个路径下自己写好zoo.cfg文件，我们已经在docker-compose.yml文件中注明了myid的内容，不需要再单独创建

zoo.cfg文件的内容：

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/data
dataLogDir=/datalog
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
autopurge.purgeInterval=1
server.1= zoo1:2888:3888
server.2= zoo2:2888:3888
server.3= zoo3:2888:3888

接下来执行docker-compose.yml文件

docker-compose up -d #后台执行

等文件执行完后doker ps查看当前运行中的容器，可以发现启动成功

接下来进入kafka容器测试收发消息：

docker exec -it kafka1 /bin/bash #进入kafka1容器

创建test主题：

bin/kafka-topics.sh --create --zookeeper zoo1:2181 --replication-factor 3 --partitions 3 --topic test

生产者发消息： 

bin/kafka-console-producer.sh --broker-list kafka1:9092 --topic test

再开一个xshell对话框，消费者收消息：（不管用哪个kafka都能收到消息才对，我们用kafka2测试收消息）

bin/kafka-console-consumer.sh --bootstrap-server Kafka:9092 --topic test --from-beginning