版本信息
组件 | 版本 |
---|---|
kafka | kafka_2.11-2.2.1 |
kafka java客户端 | kafka-clients-2.4.1.jar |
一、Kafka集群配置
定义Kafka集群认证配置文件 kafka_server_jaas.conf :
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin"
password="adminPwd"
user_admin="adminPwd"
user_testUser="testUserPwd"
};
修改 Kafka server.properties相关配置配置:
listeners=SASL_PLAINTEXT://:9092
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
super.users=User:admin
allow.everyone.if.no.acl.found=true
kafka_server_jaas.conf 配置文件中,进行认证的用户以User_开头,后面跟用户名,这里server.properties中我们使用admin用户作为broker之间进行通信所使用的用户。
修改Kafka启动脚本,在bin/kafka-server-start.sh 第一行加入配置文件的环境变量:
export KAFKA_OPTS=" -Djava.security.auth.login.config=$kafka_server_jaas_conf_file_path"
相对应的集群中的每个broker进行相同配置即可。
二、ACL授权
# 给用户 testUser 添加 testTopic 主题的 生产者权限
./kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --operation Write --topic testTopic
# 给用户 wyk_reader 添加csdn01主题的 消费者权限
#需要注意这里消费者还需要给消费者组配置权限,消费者组名称要和consumer.properties中配置的group.id一致
./kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --operation Read --topic testTopic
./kafka-acls.sh --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --consumer --topic testTopic --group testConsumer
三、java生产/消费者示例
在生产/消费程序的使用过程中有两种方式:
1、需要根据服务器端的配置定义kafka_client_jaas.conf,并将kafka_client_jaas.conf路径设置到程序系统环境变量 java.security.auth.login.config 中
2、Properties 中进行认证配置
示例如下,使用第一种,第二种为注释代码:
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="testUser"
password="testUserPwd";
};
生产者示例代码:
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringSerializer;
import java.util.Properties;
public class KafkaProducer_SASL {
private static KafkaProducer<String, String> producer = null;
public static KafkaProducer<String, String> getProducer(){
if(KafkaProducer_SASL.producer == null){
Properties properties = new Properties();
properties.put("key.serializer", StringSerializer.class.getName());
properties.put("value.serializer", StringSerializer.class.getName());
properties.put("bootstrap.servers", "hostname:port");
properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,"SASL_PLAINTEXT");
properties.put(SaslConfigs.SASL_MECHANISM,"PLAIN");
// 方式 2
// properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"testUser\" password=\"testUserPwd\";");
KafkaProducer_SASL.producer = new KafkaProducer<String,String>(properties);
}
return KafkaProducer_SASL.producer;
}
public static void main(String[] args) {
// 方式 1
System.setProperty("java.security.auth.login.config", "..\\resource\\kafka_client_jaas.conf");
String message = "testMessage";
KafkaProducer<String, String> p = KafkaProducer_SASL.getProducer();
ProducerRecord<String, String> producerRecord = new ProducerRecord<>("testTopic", message + System.lineSeparator());
p.send(producerRecord);
p.close();
}
}
消费者代码:
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Arrays;
import java.util.Properties;
public class KafkaConsumer_SASL {
private static KafkaConsumer<String, String> kafkaConsumer = null;
public static KafkaConsumer<String, String> getConsumer(){
if(KafkaConsumer_SASL.kafkaConsumer == null){
Properties properties = new Properties();
properties.put("key.deserializer", StringDeserializer.class.getName());
properties.put("value.deserializer", StringDeserializer.class.getName());
properties.put("bootstrap.servers", "hostname:port");
properties.setProperty("group.id","testConsumer");
// 方式 2
// properties.put("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"testUser\" password=\"testUserPwd\";");
properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,"SASL_PLAINTEXT");
properties.put(SaslConfigs.SASL_MECHANISM,"PLAIN");
KafkaConsumer_SASL.kafkaConsumer = new KafkaConsumer<String, String>(properties);
}
return KafkaConsumer_SASL.kafkaConsumer;
}
public static void main(String[] args) {
// 方式 1
System.setProperty("java.security.auth.login.config", "..\\resource\\kafka_client_jaas.conf");
KafkaConsumer<String, String> consumer = KafkaConsumer_SASL.getConsumer();
consumer.subscribe(Arrays.asList("testTopic".split(",")));
while( true ){
for(ConsumerRecord<String,String> record : consumer.poll(30000) ){
System.out.print(record.value());
}
}
}
}