kafka 3.4.0(kraft) 集群+ kowl

一、环境说明

IP	操作系统	服务
192.168.11.100	CentOs 7	kafka , kowl
192.168.11.101	CentOs 7	kafka
192.168.11.102	CentOs 7	kafka

二、安装docker
略。。。

三、安装kafka

path=/data/kafka
mkdir -p ${path}/{data,etc,log}
chown -R 1001 ${path}

cat > ${path}/etc/kafka_jaas.conf << 'EOF'
KafkaClient {
   org.apache.kafka.common.security.plain.PlainLoginModule required
   username="gohangout"
   password="passwordXXXXX";
   };
KafkaServer {
   org.apache.kafka.common.security.plain.PlainLoginModule required
   username="inuser"
   password="BitNami"
   user_inuser="BitNami"
   user_gohangout="passwordXXXXX";
   org.apache.kafka.common.security.scram.ScramLoginModule required;
   };
EOF

cat > ${path}/etc/sasl_config.properties << 'EOF'
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="clickhouse" password="passwordXXXXX";
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
EOF


# KAFKA_BROKER_ID 、 KAFKA_CFG_ADVERTISED_LISTENERS  、 KAFKA_CFG_CONTROLLER_QUORUM_VOTERS 根据实际情况填写
cat > ${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`

docker run -d \
--name kafka \
--restart=always \
--network host \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_HEAP_OPTS="-Xmx512m -Xms512m" \
-e KAFKA_ENABLE_KRAFT=yes \
-e KAFKA_CFG_PROCESS_ROLES=broker,controller \
-e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:SASL_PLAINTEXT,CONTROLLER:PLAINTEXT \
-e KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 \
-e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://192.168.11.100:9092 \
-e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@192.168.11.100:9093,2@192.168.11.101:9093,3@192.168.11.102:9093 \
-e KAFKA_KRAFT_CLUSTER_ID="Aqvf7RVETX-DInZbNUS2Wg" \
-e KAFKA_CFG_SASL_ENABLED_MECHANISMS="PLAIN" \
-e KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL="PLAIN" \
-e ALLOW_PLAINTEXT_LISTENER=yes \
-e KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true \
-e KAFKA_LOG_RETENTION_HOURS=24 \
-v `pwd`/etc/kafka_jaas.conf:/opt/bitnami/kafka/config/kafka_jaas.conf \
-v `pwd`/etc/sasl_config.properties:/opt/bitnami/kafka/config/sasl_config.properties \
-v `pwd`/data:/bitnami/kafka/ \
-v /etc/localtime:/etc/localtime \
bitnami/kafka:3.4.0
EOF

bash ${path}/start.sh

docker exec -it kafka bash 

#创建topic
kafka-topics.sh --create --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test  --command-config /opt/bitnami/kafka/config/sasl_config.properties

#生产
kafka-console-producer.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test --producer.config /opt/bitnami/kafka/config/sasl_config.properties
#消费
kafka-console-consumer.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test --consumer.config /opt/bitnami/kafka/config/sasl_config.properties

#扩容分区
kafka-topics.sh --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --alter --topic  log-smartgate --partitions 3 --replication-factor 1 --command-config /opt/bitnami/kafka/config/sasl_config.properties

#查询分区
kafka-topics.sh --describe --bootstrap-server 192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092 --topic test  --command-config /opt/bitnami/kafka/config/sasl_config.properties

四、安装kowl

path=/data/kowl
mkdir -p ${path}

cat > ${path}/start.sh << 'EOF'
#!/bin/bash
docker rm -f kowl

cd $(dirname $0)
docker run -itd \
--restart=always \
--name kowl \
-v /etc/localtime:/etc/localtime \
-p 19002:8080 \
-e  KAFKA_BROKERS="192.168.11.100:9092,192.168.11.101:9092,192.168.11.102:9092" \
-e KAFKA_TLS_ENABLED=false \
-e KAFKA_SASL_ENABLED=true \
-e KAFKA_SASL_USERNAME=gohangout \
-e KAFKA_SASL_PASSWORD="passwordXXXXX" \
rsmnarts/kowl:latest
EOF

bash ${path}/start.sh