kafka集群及SASL认证设置

server.properties

#listeners=SASL_PLAINTEXT://172.19.115.100（0.0.0.0）:9092
listeners=PLAINTEXT://172.19.115.100:9092
advertised.listeners=PLAINTEXT://172.19.115.100:9092
#SASL开始
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN


allow.everyone.if.no.acl.found=false
#超级管理员权限用户
super.users=User:admin
advertised.listeners=SASL_PLAINTEXT://172.19.115.100:9092
#结束

zookeeper.properties

dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
# Disable the adminserver by default to avoid port conflicts.
# Set the port to something non-conflicting if choosing to enable this
admin.enableServer=false
# admin.serverPort=8080
tickTime=2000
initLimit=10
syncLimit=5
server.0=172.19.115.100:2888:3888
server.1=172.19.115.98:2888:3888
server.2=172.19.115.99:2888:3888


#SASL开始
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
#zookeeper.sasl.client=true
#结束

kafka_server_jaas.conf

KafkaServer {
 org.apache.kafka.common.security.plain.PlainLoginModule required
 username="admin"
 password="admin"
 user_admin="admin"
 user_producer="producer@123"
 user_consumer="consumer@123";
};

Client {
 org.apache.kafka.common.security.plain.PlainLoginModule required
 username="admin"
 password="admin";
};

consumer.properties

bootstrap.servers=localhost:9092

# consumer group id
group.id=test-consumer-group

#SASL开始
##username 和 password 对应kafka_server_jaas.conf中的用户名密码
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin";
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN

producer.properties

bootstrap.servers=localhost:9092

# specify the compression codec for all data generated: none, gzip, snappy, lz4, zstd
compression.type=none

#SASL开始
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="producer" password="producer@123";
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN

zookeeper.properties

dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
# Disable the adminserver by default to avoid port conflicts.
# Set the port to something non-conflicting if choosing to enable this
admin.enableServer=false
# admin.serverPort=8080
tickTime=2000
initLimit=10
syncLimit=5
server.0=172.19.115.100:2888:3888
server.1=172.19.115.98:2888:3888
server.2=172.19.115.99:2888:3888


#SASL开始
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
#zookeeper.sasl.client=true

kafka_consumer_jaas.conf

Client { 
org.apache.kafka.common.security.plain.PlainLoginModule required 
username="consumer" 
password="consumer@123"; 
};

kafka_producer_jaas.conf

Client { 
org.apache.kafka.common.security.plain.PlainLoginModule required 
username="producer" 
password="producer@123"; 
};

zoo_jaas.conf

ZKServer {
  org.apache.kafka.common.security.plain.PlainLoginModule required
  username="admin"
  password="admin"
  user_admin="admin";
};

sasl.properties

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin";

bin命令

#zookeeper-server-start.sh

export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/zoo_jaas.conf"
#exec $base_dir/kafka-run-class.sh $EXTRA_ARGS -Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/zoo_jaas.conf org.apache.zookeeper.server.quorum.QuorumPeerMain "$@"
exec $base_dir/kafka-run-class.sh $EXTRA_ARGS  org.apache.zookeeper.server.quorum.QuorumPeerMain "$@"


#kafka-server-start.sh

export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_server_jaas.conf"
exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka "$@"
#exec $base_dir/kafka-run-class.sh $EXTRA_ARGS -Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_server_jaas.conf kafka.Kafka "$@"


#kafka-console-consumer.sh

export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_server_jaas.conf"
#exec $(dirname $0)/kafka-run-class.sh -Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_consumer_jaas.conf  kafka.tools.ConsoleConsumer "$@"
exec $(dirname $0)/kafka-run-class.sh  kafka.tools.ConsoleConsumer "$@"



#kafka-console-producer.sh

export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_server_jaas.conf"
#exec $(dirname $0)/kafka-run-class.sh -Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_producer_jaas.conf  kafka.tools.ConsoleProducer "$@"
exec $(dirname $0)/kafka-run-class.sh kafka.tools.ConsoleProducer "$@"



#kafka-topics.sh

export KAFKA_OPTS="-Xmx1G -Xms1G -Djava.security.auth.login.config=/usr/local/kafka_2.13-3.1.0/config/kafka_server_jaas.conf"
exec $(dirname $0)/kafka-run-class.sh kafka.admin.TopicCommand "$@"

kafka命令

./bin/kafka-console-producer.sh --broker-list 172.19.115.100:9092 --topic topic001 -producer.config ./config/producer.properties


./bin/kafka-console-consumer.sh --bootstrap-server 172.19.115.100:9092 --topic topic-test --from-beginning --consumer.config ./config/consumer.properties
./bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic topic-test --from-beginning --consumer.config ./config/consumer.properties

./bin/kafka-console-consumer.sh --bootstrap-server 172.19.115.100:9092 --topic test --from-beginning --consumer.config ./config/consumer.properties

 
./bin/kafka-topics.sh --create --bootstrap-server 172.19.115.100:9092 --replication-factor 3 --partitions 1 --topic test123


./bin/kafka-topics.sh --list --bootstrap-server localhost:9092 --command-config ./config/sasl.properties


cd /usr/local/kafka_2.13-3.1.0/

./bin/zookeeper-server-start.sh ./config/zookeeper.properties

./bin/kafka-server-start.sh ./config/server.properties