搭建zookeeper:
docker run -d -p 2181:2181 -v /mysoft/zookeeper/data/:/data/ --name=zookeeper --privileged xqkang/zookeeper:v1
46节点
sysuser@qj-pred-pub135:/qj/kafka1$ cat run.sh
#!/bin/bash
zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka1 --network host -d \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_ZOOKEEPER_CONNECT=${zks} \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.46:9092 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-v /qj/kafka/log:/kafka \
--restart=always \
xqkang/kafka:2.12-2.2.2
150节点
sysuser@qj-pred-pub99:/qj/kafka2$ cat run.sh
#!/bin/bash
zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka2 --network host -d \
-e KAFKA_BROKER_ID=2 \
-e KAFKA_ZOOKEEPER_CONNECT=${zks} \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.150:9092 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-v /qj/kafka/log:/kafka \
--restart=always \
xqkang/kafka:2.12-2.2.2
207节点
sysuser@qj-pred-pub123:/qj/kafka3$ cat run.sh
#!/bin/bash
zks="192.168.10.46:2181,192.168.10.150:2181,192.168.10.207:2181"
docker run -p 9092:9092 --name kafka2 --network host -d \
-e KAFKA_BROKER_ID=3 \
-e KAFKA_ZOOKEEPER_CONNECT=${zks} \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.207:9092 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-v /qj/kafka/log:/kafka \
--restart=always \
xqkang/kafka:2.12-2.2.2
创建topic和生产者:
docker exec -it kafka bash
cd /opt/kafka_2.12-2.2.2/bin/
./kafka-topics.sh --create --zookeeper 192.168.10.46:2181 --replication-factor 1 --partitions 8 --topic test
./kafka-console-producer.sh --broker-list localhost:9092 --topic test
创建消费者:
docker exec -it kafka bash
cd /opt/kafka_2.11-2.0.0/bin/
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
参考:https://www.jianshu.com/p/cec64efdf1e0
删除topic
查看复制打印?
[root@namenode1 kafka]# bin/kafka-run-class.sh kafka.admin.TopicCommand --delete --topic track_pc --zookeeper namenode1:2181,namenode2:2181,datanode1:2181
补充:
内外网通用:
在远程访问的主机上
vim /etc/hosts
xxxx(外网IP) broker1
xxxx(外网IP) broker2
xxxx(外网IP) broker3
189:
[sysuser@server189 ~]$ cat /qj/kafka/run.sh
#!/bin/bash
sudo docker run -p 9092:9092 --name kafka1 -d \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_ADVERTISED_HOST_NAME=broker1 \
-e KAFKA_ADVERTISED_PORT=9092 \
-e KAFKA_HOST_NAME=broker1 \
-e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
-e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9092" \
-e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker1:9092" \
--add-host broker1:192.168.10.189 \
--add-host broker2:192.168.10.191 \
--add-host broker3:192.168.10.192 \
--add-host zk1:192.168.10.189 \
--add-host zk2:192.168.10.191 \
--add-host zk3:192.168.10.192 \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /qj/kafka/log:/kafka \
--restart=always \
registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2
191:
[sysuser@server191 ~]$ cat /qj/kafka/run.sh
#!/bin/bash
sudo docker run -p 9093:9093 --name kafka2 -d \
-e KAFKA_BROKER_ID=2 \
-e KAFKA_ADVERTISED_HOST_NAME=broker2 \
-e KAFKA_ADVERTISED_PORT=9093 \
-e KAFKA_HOST_NAME=broker2 \
-e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
-e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9093" \
-e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker2:9093" \
--add-host broker1:192.168.10.189 \
--add-host broker2:192.168.10.191 \
--add-host broker3:192.168.10.192 \
--add-host zk1:192.168.10.189 \
--add-host zk2:192.168.10.191 \
--add-host zk3:192.168.10.192 \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /qj/kafka/log:/kafka \
--restart=always \
registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2
192:
[sysuser@server192 ~]$ cat /qj/kafka/run.sh
#!/bin/bash
sudo docker run -p 9094:9094 --name kafka3 -d \
-e KAFKA_BROKER_ID=3 \
-e KAFKA_ADVERTISED_HOST_NAME=broker3 \
-e KAFKA_ADVERTISED_PORT=9094 \
-e KAFKA_HOST_NAME=broker3 \
-e KAFKA_ZOOKEEPER_CONNECT=zk1:2181,zk2:2181,zk3:2181 \
-e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:9094" \
-e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://broker3:9094" \
--add-host broker1:192.168.10.189 \
--add-host broker2:192.168.10.191 \
--add-host broker3:192.168.10.192 \
--add-host zk1:192.168.10.189 \
--add-host zk2:192.168.10.191 \
--add-host zk3:192.168.10.192 \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /qj/kafka/log:/kafka \
--restart=always \
registry.cn-beijing.aliyuncs.com/qianjia_public/kafka:2.12-2.2.2
创建topic:
./kafka-topics.sh --create --bootstrap-server broker1:9092,broker2:9093,broker3:9094 --replication-factor 2 --partitions 3 --topic kkk
生产:
./kafka-console-producer.sh --broker-list broker1:9092,broker2:9093,broker3:9094 --topic kkk
消费:
./kafka-console-consumer.sh --bootstrap-server broker1:9092,broker2:9093,broker3:9094 --topic kkk --from-beginning