kafka 3.4.0 集群开启 sasl 认证

kafka集群

记得装jdk

搭建

以下操作3台都要做

下载安装包，解压

cd /data
wget https://mirrors.cloud.tencent.com/apache/kafka/3.4.0/kafka_2.13-3.4.0.tgz
tar zxvf kafka_2.13-3.4.0.tgz 
mv kafka* kafka
#修改jvm参数
sed -i 's/-Xmx1G -Xms1G/-Xmx4G -Xms4G/' /data/kafka/bin/kafka-server-start.sh

配置环境变量

cat <<'EOF'> /etc/profile.d/kafka.sh
export KAFKA_HOME=/data/kafka
export PATH=$PATH:$KAFKA_HOME/bin
EOF

source /etc/profile.d/kafka.sh

重启脚本

cat <<EOF> /data/kafka/restart.sh
#!/bin/bash

kafka-server-stop.sh
nohup kafka-server-start.sh config/server.properties >> /data/kafka/nohup.out 2>&1 &
EOF

chmod +x /data/kafka/restart.sh

用于监控的配置，修改 bin/kafka-server-start.sh，增加 JMX_PORT，可以获取更多指标。

if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
    export KAFKA_HEAP_OPTS="-Xmx4G -Xms4G"
    export JMX_PORT="9099"
fi

关键配置如下：

name	含义	举例
broker.id	一个Kafka节点就是一个Broker.id，要保证唯一性	broker.id=0
listeners	kafka只面向内网时用到listeners，内外网需要作区分时才需要用到advertised.listeners	listeners=PLAINTEXT://192.168.0.137:9092
zookeeper.connect	配置zk集群信息	zookeeper.connect=192.168.0.19:2181,192.168.0.36:2181,192.168.0.18:2181

kafka01 配置文件/data/kafka/config/server.properties

broker.id=0
# sasl
listeners=SASL_PLAINTEXT://192.168.96.158:9092
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.authorizer.AclAuthorizer
super.users=User:admin
# sasl
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.96.158:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

kafka02 配置文件/data/kafka/config/server.properties

broker.id=1
listeners=SASL_PLAINTEXT://192.168.96.134:9092
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.authorizer.AclAuthorizer
super.users=User:admin
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.96.158:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

kafka03 配置文件/data/kafka/config/server.properties

broker.id=2
listeners=SASL_PLAINTEXT://192.168.96.31:9092
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.authorizer.AclAuthorizer
super.users=User:admin
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.96.158:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

config 目录下新建 2 个文件

cat <<EOF> /data/kafka/config/kafka_server_jaas.conf
KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="admin"
    password="6aJSMoftVgMC"
    user_admin="6aJSMoftVgMC"
    user_alice="6aJSMoftVgMC";
};
EOF

cat <<EOF> /data/kafka/config/kafka_client_jaas.conf
KafkaClient {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="admin"
    password="6aJSMoftVgMC";
};
EOF

修改启动参数

sed -i '$i\export KAFKA_OPTS="-Djava.security.auth.login.config=/data/kafka/config/kafka_server_jaas.conf"' /data/kafka/bin/kafka-server-start.sh

sed -i '$i\export KAFKA_OPTS="-Djava.security.auth.login.config=/data/kafka/config/kafka_client_jaas.conf"' /data/kafka/bin/kafka-console-consumer.sh

sed -i '$i\export KAFKA_OPTS="-Djava.security.auth.login.config=/data/kafka/config/kafka_client_jaas.conf"' /data/kafka/bin/kafka-console-producer.sh

3台都启动

/data/kafka/restart.sh