docker 搭建kafka集群

搭建zookeeper:

docker run -d -p 2181:2181 -v /mysoft/zookeeper/data/:/data/ --name=zookeeper  --privileged xqkang/zookeeper:v1

46节点

sysuser@qj-pred-pub135:/qj/kafka1$ cat run.sh 
#!/bin/bash

zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka1 --network host -d \
  -e KAFKA_BROKER_ID=1 \
  -e KAFKA_ZOOKEEPER_CONNECT=${zks} \
  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.46:9092 \
  -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
  -v /qj/kafka/log:/kafka \
  --restart=always \
  xqkang/kafka:2.12-2.2.2

150节点

sysuser@qj-pred-pub99:/qj/kafka2$ cat run.sh 
#!/bin/bash

zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka2 --network host -d \
  -e KAFKA_BROKER_ID=2 \
  -e KAFKA_ZOOKEEPER_CONNECT=${zks} \
  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.150:9092 \
  -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
  -v /qj/kafka/log:/kafka \
  --restart=always \
  xqkang/kafka:2.12-2.2.2

207节点

sysuser@qj-pred-pub123:/qj/kafka3$ cat run.sh 
#!/bin/bash

zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka2 --network host -d \
  -e KAFKA_BROKER_ID=3 \
  -e KAFKA_ZOOKEEPER_CONNECT=${zks} \
  -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.207:9092 \
  -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
  -v /qj/kafka/log:/kafka \
  --restart=always \
xqkang/kafka:2.12-2.2.2

创建topic和生产者:

docker exec -it kafka bash
cd /opt/kafka_2.12-2.2.2/bin/
./kafka-topics.sh --create --zookeeper 192.168.10.46:2181 --replication-factor 1 --partitions 8 --topic test
./kafka-console-producer.sh --broker-list localhost:9092 --topic test

创建消费者:

docker exec -it kafka bash
cd /opt/kafka_2.11-2.0.0/bin/
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning

参考:https://www.jianshu.com/p/cec64efdf1e0

删除topic
查看复制打印?

[root@namenode1 kafka]# bin/kafka-run-class.sh kafka.admin.TopicCommand --delete --topic track_pc --zookeeper namenode1:2181,namenode2:2181,datanode1:2181 

补充：

内外网通用：
在远程访问的主机上

vim /etc/hosts
xxxx(外网IP)  broker1
xxxx(外网IP)  broker2
xxxx(外网IP)  broker3

189:

[sysuser@server189 ~]$ cat /qj/kafka/run.sh
#!/bin/bash

sudo docker run -p 9092:9092 --name kafka1 -d \
  -e KAFKA_BROKER_ID=1 \
  -e KAFKA_ADVERTISED_HOST_NAME=broker1 \
  -e KAFKA_ADVERTISED_PORT=9092 \
  -e KAFKA_HOST_NAME=broker1 \
  -e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
  -e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9092" \
  -e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker1:9092" \
  --add-host broker1:192.168.10.189 \
  --add-host broker2:192.168.10.191 \
  --add-host broker3:192.168.10.192 \
  --add-host zk1:192.168.10.189 \
  --add-host zk2:192.168.10.191 \
  --add-host zk3:192.168.10.192 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /qj/kafka/log:/kafka \
  --restart=always \
  registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2

191:

[sysuser@server191 ~]$ cat /qj/kafka/run.sh 
#!/bin/bash

sudo docker run -p 9093:9093 --name kafka2 -d \
  -e KAFKA_BROKER_ID=2 \
  -e KAFKA_ADVERTISED_HOST_NAME=broker2 \
  -e KAFKA_ADVERTISED_PORT=9093 \
  -e KAFKA_HOST_NAME=broker2 \
  -e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
  -e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9093" \
  -e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker2:9093" \
  --add-host broker1:192.168.10.189 \
  --add-host broker2:192.168.10.191 \
  --add-host broker3:192.168.10.192 \
  --add-host zk1:192.168.10.189 \
  --add-host zk2:192.168.10.191 \
  --add-host zk3:192.168.10.192 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /qj/kafka/log:/kafka \
  --restart=always \
  registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2

192:

[sysuser@server192 ~]$  cat /qj/kafka/run.sh
#!/bin/bash

sudo docker run -p 9094:9094 --name kafka3 -d \
  -e KAFKA_BROKER_ID=3 \
  -e KAFKA_ADVERTISED_HOST_NAME=broker3 \
  -e KAFKA_ADVERTISED_PORT=9094 \
  -e KAFKA_HOST_NAME=broker3 \
  -e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
  -e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9094" \
  -e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker3:9094" \
  --add-host broker1:192.168.10.189 \
  --add-host broker2:192.168.10.191 \
  --add-host broker3:192.168.10.192 \
  --add-host zk1:192.168.10.189 \
  --add-host zk2:192.168.10.191 \
  --add-host zk3:192.168.10.192 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /qj/kafka/log:/kafka \
  --restart=always \
  registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2

创建topic:

./kafka-topics.sh --create --bootstrap-server broker1:9092,broker2:9093,broker3:9094 --replication-factor 2 --partitions 3 --topic kkk

生产：

 ./kafka-console-producer.sh --broker-list broker1:9092,broker2:9093,broker3:9094  --topic kkk

消费：

./kafka-console-consumer.sh --bootstrap-server broker1:9092,broker2:9093,broker3:9094 --topic kkk --from-beginning