docker-compose部署kafka集群

Kafka官网：https://kafka.apache.org/

2.1 部署环境准备

linux环境、docker环境

部署环境配置	版本	下载链接
ECS	CentOS 7.6	三台2vCPUs|4GiB
docker	19.03.9	脚本
docker-compose	2.21.0	sudo curl -L https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose

2.2 服务器信息

ID	规格	IP
ecs-kafka1	centos7.6,2vCPUs|4GiB	172.16.20.92
ecs-kafka2	centos7.6,2vCPUs|4GiB	172.16.20.207
ecs-kafka3	centos7.6,2vCPUs|4GiB	172.16.20.209

2.3 脚本

#!/bin/bash
##################### 准备工作 #####################
#1.镜像拉取速度慢的话加镜像加速器脚本
#2.镜像加速器地址https://console.huaweicloud.com/swr/?region=cn-north-4#/swr/mirror
#3.注意修改IP地址&&根据自己需要修改挂载文件和文件变量
#4.修改broker的id与zk的myid
#5.修改zk与kafka的命名与端口

#下载docker-compose二进制文件
#sudo curl -L https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose

#创建配置文件
mkdir -p /data/wangzunbin/volume/zkcluster/zoo1/data
mkdir -p /data/wangzunbin/volume/zkcluster/zoo1/datalog
mkdir -p /data/wangzunbin/volume/kfkluster/kafka1/logs
mkdir -p /data/deploy/kafkaCluster
mkdir -p /secrets

#宿主机文件
file1=/data/wangzunbin/volume/zkcluster/zoo1/data
file2=/data/wangzunbin/volume/zkcluster/zoo1/datalog
file3=/data/wangzunbin/volume/kfkluster/kafka1/logs
file4=/data/deploy/kafkaCluster
file5=/secrets

#映射容器文件
wenjian1=/data
wenjian2=/datalog
wenjian3=/kafka
wenjian4=/etc/kafka/secrets

#IP变量
ip1=172.16.20.74
ip2=172.16.20.142
ip3=172.16.20.232

#更改brokerid=0.1.2
VAR=2 #节点id修改

#zookeeper节点的标记1,2,3
myid=3           #zookeeper节点标记修改

#更改监控ip变量ip1,ip2,ip3
listeners="$ip3" 

#zk命名zoo1,zoo2,zoo3
zk=zoo3

#zk端口2181，2182，2183
port=2183

#kafka命名kafka1,kafka2,kafka3
ka=kafka3

#kafka端口9092,9093,9094
kport=9094

##################################################################################

#创建网络
docker network create app-tier 

echo 'Server {' >> $file5/zk_server_jaas.conf
sed -i -e '$a org.apache.kafka.server.auth.DigestLoginModule required \nusername="kafka" \npassword="kafka” \nuser_kafka="kafka"; \n};' $file5/zk_server_jaas.conf

echo 'KafkaServer {' >> $file5/kafka_server_jaas.conf
sed -i -e 'aorg.apache.kafka.common.security.plain.PlainLoginModule required \nusername="admin" \npassword="admin" \nuser_admin="admin" \nuser_kafka="kafka"; \n}; \nClient { \norg.apache.kafka.server.auth.DigestLoginModule required \nusername="kafka" \npassword="kafka"; \n};' $file5/kafka_server_jaas.conf

echo 'KafkaClient {' >> $file5/kafka_client_jaas.conf
sed -i -e 'aorg.apache.kafka.common.security.plain.PlainLoginModule required \nusername="admin" \npassword="admin"; \n}; \nClient { \norg.apache.kafka.server.auth.DigestLoginModule required \nusername="admin" \npassword="admin"; \n};' $file5/kafka_client_jaas.conf

echo -e "security.protocol=SASL_PLAINTEXT \nsasl.mechanism=PLAIN" >> $file5/consumer.properties
echo -e "security.protocol=SASL_PLAINTEXT \nsasl.mechanism=PLAIN" >> $file5/producer.properties

#Topic配置sasl文件
echo 'sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin";' >> $file5/config.properties
sed -i -e '$a security.protocol=SASL_PLAINTEXT \nsasl.mechanism=PLAIN' $file5/config.properties

cd $file4
cat > docker-compose.yml <<-EOF
version: '3.1'
services:
  $zk:
    image: wurstmeister/zookeeper
    restart: always
    hostname: $zk
    container_name: $zk
    ports:
      - $port:2181
    volumes:
      - $file1:$wenjian1:Z
      - $file2:$wenjian2:Z
      - $file5:$wenjian4:Z
    environment:
      ZOO_MY_ID: $myid
      ZOO_SERVERS: server.1=$ip1:2888:3888;2181 server.2=$ip2:2888:3888;2181 server.3=$ip3:2888:3888;2181
      ZOOKEEPER_AUTHPROVIDER.1: org.apache.zookeeper.server.auth.SASLAuthenticationProvider
      ZOOKEEPER_REQUIRECLIENTAUTHSCHEME: sasl
      ZOOKEEPER_JAASLOGINRENEW: 3600000
      KAFKA_OPTS: -Djava.security.auth.login.config=$wenjian4/zk_server_jaas.conf       
    network_mode: app-tier
    
  $ka:
    image: wurstmeister/kafka
    restart: always
    hostname: $ka
    container_name: $ka
    ports:
      - $kport:9092
    environment:
      KAFKA_ADVERTISED_HOST_NAME: $listeners
      KAFKA_HOST_NAME: $listeners
      KAFKA_ADVERTISED_PORT: $kport
      KAFKA_BROKER_ID: $VAR
      KAFKA_ZOOKEEPER_CONNECT: $ip1:2181,$ip2:2181,$ip3:2181
      KAFKA_LISTENERS: SASL_PLAINTEXT://0.0.0.0:$kport
      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://$listeners:$kport
      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SASL_PLAINTEXT
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_OPTS: -Djava.security.auth.login.config=$wenjian4/kafka_server_jaas.conf
      KAFKA_SUPER_USERS: User:admin
    volumes:
      - $file3:$wenjian3:Z
      - $file5:$wenjian4:Z
    network_mode: app-tier
EOF
docker-compose up -d