Kafka SASL认证/消费和生产者使用示例

版本信息

组件	版本
kafka	kafka_2.11-2.2.1
kafka java客户端	kafka-clients-2.4.1.jar

一、Kafka集群配置

定义Kafka集群认证配置文件 kafka_server_jaas.conf ：

KafkaServer {
        org.apache.kafka.common.security.plain.PlainLoginModule required
        username="admin"
        password="adminPwd"
        user_admin="adminPwd"
        user_testUser="testUserPwd"
};

修改 Kafka server.properties相关配置配置：

listeners=SASL_PLAINTEXT://:9092

security.inter.broker.protocol=SASL_PLAINTEXT
sasl.enabled.mechanisms=PLAIN
sasl.mechanism.inter.broker.protocol=PLAIN
authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer
super.users=User:admin
allow.everyone.if.no.acl.found=true

kafka_server_jaas.conf 配置文件中，进行认证的用户以User_开头，后面跟用户名，这里server.properties中我们使用admin用户作为broker之间进行通信所使用的用户。

修改Kafka启动脚本，在bin/kafka-server-start.sh 第一行加入配置文件的环境变量：

export KAFKA_OPTS=" -Djava.security.auth.login.config=$kafka_server_jaas_conf_file_path"

相对应的集群中的每个broker进行相同配置即可。

二、ACL授权

# 给用户 testUser 添加 testTopic 主题的 生产者权限
./kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --operation Write --topic testTopic

# 给用户 wyk_reader 添加csdn01主题的 消费者权限
#需要注意这里消费者还需要给消费者组配置权限，消费者组名称要和consumer.properties中配置的group.id一致
./kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --operation Read --topic testTopic

./kafka-acls.sh --authorizer-properties zookeeper.connect=z01:2181 --add --allow-principal User:testUser --consumer --topic testTopic --group testConsumer

三、java生产/消费者示例

在生产/消费程序的使用过程中有两种方式:
1、需要根据服务器端的配置定义kafka_client_jaas.conf，并将kafka_client_jaas.conf路径设置到程序系统环境变量 java.security.auth.login.config 中
2、Properties 中进行认证配置

示例如下，使用第一种，第二种为注释代码：

KafkaClient {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="testUser"
    password="testUserPwd";
};

生产者示例代码：

import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;

public class KafkaProducer_SASL {
	
	private static KafkaProducer<String, String> producer = null;
	
	public static KafkaProducer<String, String> getProducer(){
		if(KafkaProducer_SASL.producer == null){
			Properties properties = new Properties();
			properties.put("key.serializer", StringSerializer.class.getName());
			properties.put("value.serializer", StringSerializer.class.getName());
	        properties.put("bootstrap.servers", "hostname:port");
			properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,"SASL_PLAINTEXT");
			properties.put(SaslConfigs.SASL_MECHANISM,"PLAIN");
			// 方式 2 
			// properties.put("sasl.jaas.config",					"org.apache.kafka.common.security.plain.PlainLoginModule required  username=\"testUser\" password=\"testUserPwd\";");
	        KafkaProducer_SASL.producer = new KafkaProducer<String,String>(properties);
		}
		return KafkaProducer_SASL.producer;
	}


	public static void main(String[] args) {
		// 方式 1
		System.setProperty("java.security.auth.login.config", "..\\resource\\kafka_client_jaas.conf");
		String message = "testMessage";
		KafkaProducer<String, String> p = KafkaProducer_SASL.getProducer();
		ProducerRecord<String, String> producerRecord = new ProducerRecord<>("testTopic", message + System.lineSeparator());
		p.send(producerRecord);
		p.close();

	}
}

消费者代码：

import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.util.Arrays;
import java.util.Properties;

public class KafkaConsumer_SASL {
	
	private static KafkaConsumer<String, String> kafkaConsumer = null;
	
	public static KafkaConsumer<String, String> getConsumer(){
		if(KafkaConsumer_SASL.kafkaConsumer == null){
			Properties properties = new Properties();
			properties.put("key.deserializer", StringDeserializer.class.getName());
			properties.put("value.deserializer", StringDeserializer.class.getName());
			properties.put("bootstrap.servers", "hostname:port");
			properties.setProperty("group.id","testConsumer");
			// 方式 2 
			// properties.put("sasl.jaas.config",					"org.apache.kafka.common.security.plain.PlainLoginModule required  username=\"testUser\" password=\"testUserPwd\";");
			properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,"SASL_PLAINTEXT");
			properties.put(SaslConfigs.SASL_MECHANISM,"PLAIN");
			KafkaConsumer_SASL.kafkaConsumer = new KafkaConsumer<String, String>(properties);
		}
		return KafkaConsumer_SASL.kafkaConsumer;
	}


	public static void main(String[] args) {
		// 方式 1
		System.setProperty("java.security.auth.login.config", "..\\resource\\kafka_client_jaas.conf");
		KafkaConsumer<String, String> consumer = KafkaConsumer_SASL.getConsumer();
		consumer.subscribe(Arrays.asList("testTopic".split(",")));

		while( true ){
			for(ConsumerRecord<String,String> record : consumer.poll(30000) ){
				System.out.print(record.value());
			}
		}

	}
	
}

参考：Kafka 2.2 Documentation