1.非spring项目配置kafka
很多时候,kafka服务都是单纯的数据清洗服务(比如单纯消息异步监听处理,非关键性业务剥离,大数据流式处理等)或者监听报警服务(比如限流,削峰,整合flink监听报警等),其实这个时候是不需要依赖spring的,可以简化服务配置。
1. 在resources目录下创建kafka配置文件(根据不同环境进行配置文件的编写,kafka-test.properties):
#需要消费的topic
topic=bi-data-sam
#消费topic下的具体某一个groupId
group.id=external-action-forwarder-sam
#kafka服务地址
bootstrap.servers=ckafka-floxf08f.ap-guangzhou.ckafka.tencentcloudmq.com:6018
#kafka账号用户名和密码
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="ckafka-floxf08f#k2t-qshare" password="Z5uH4prcr7KGqTEg6PjKBqqCqLcXpZyD";
#安全认证方式 kafka所有认证方式可以参考https://blog.csdn.net/u012036736/article/details/84747923
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
#默认值是 true;就是默认采用自动提交的机制,建议关闭,自行提交
enable.auto.commit=false
#注册consumer后offset从哪里开始消费,earliest是从最后提交的offset开始,没有就从头开始消费;
# latest是当各分区下有已提交的offset时,从提交的offset开始消费;无提交的offset时,消费新产生的该分区下的数据;
# none是当topic各分区都存在已提交的offset时,从offset后开始消费;只要有一个分区不存在已提交的offset,则抛出异常
auto.offset.reset=earliest
#超时时间
session.timeout.ms=20000
#用于指定consumer两次poll的最大时间间隔(默认5分钟)
max.poll.interval.ms=500000
#单次拉取消息的最大长度(默认500)
max.poll.records=2000
2. 读取配置文件,加载Properties:
private final Properties config = new Properties();
protected void configure() {
try (InputStream inputStream =
Thread.currentThread()
.getContextClassLoader()
.getResourceAsStream(getKafkaPropertiesName(name))) {
assert inputStream != null;
config.load(inputStream);
} catch (IOException e) {
throw new RuntimeException(e);
}
Map<String, String> topicAndGroup =
ImmutableMap.of("kafka.topic.list", (String) config.remove("topic"));
}
/**
* 根据环境确定读取什么配置文件
* @return
*/
String getKafkaPropertiesName(String name) {
return String.format("kafka-%s.properties", name);
}
3. 创建kafka consumer,一定要保证单例:
@Singleton
KafkaConsumer<byte[], byte[]> provideKafkaConsumer() {
return new KafkaConsumer<>(
config, Serdes.ByteArray().deserializer(), Serdes.ByteArray().deserializer());
}
4. 针对consumer rebalance进行再平衡的时候的特殊处理:
@RequiredArgsConstructor
private static final class HandleRebalance implements ConsumerRebalanceListener {
private final KafkaConsumer<?, ?> consumer;
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
log.warn("Lost partitions in rebalance, committing current offsets.");
try {
consumer.commitSync();
} catch (Exception e) {
log.error("Committing offsets failed: ", e);
}
}
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
log.info("Partitions assigned: {}", partitions);
}
}
5. 监听kafka topic,拉取消息并解析:
//获取kafka消费者单例
private final KafkaConsumer<byte[], byte[]> kafkaConsumer;
public static void main(String[] args) {
log.info(
//当前环境从变量传入
EnvType envType = EnvType.fromSystem();
//控制程序启动
AtomicBoolean stopped = new AtomicBoolean(false);
//程序shutdown之后置换变量,将程序强制结束,防止shuadown之后main函数不立即停止导致的问题
Runtime.getRuntime()
.addShutdownHook(Executors.defaultThreadFactory().newThread(() -> stopped.set(true)));
try {
app.start(stopped);
} catch (Exception e) {
log.error("App raise exception: ", e);
} finally {
//报错后直接停止consumer,根据实际业务逻辑来
app.close();
}
}
private void start(AtomicBoolean stopped) {
log.info("Starting kafka consumer app...");
//注册服务
kafkaConsumer.subscribe(
Arrays.asList(StringUtils.split(kafkaTopicList, ',')), new HandleRebalance(kafkaConsumer));
while (!stopped.get()) {
//从group中拉取数据
ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(TIMEOUT);
if (records.isEmpty()) continue;
monitor.reportAsync(MonitorType.MSG_RECEIVE, records.count());
List<byte[]> bytes = new ArrayList<>(records.count());
for (ConsumerRecord<byte[], byte[]> record : records) {
bytes.add(record.value());
}
List<byte[]> bytesAfterMapping = mappingFunction.apply(bytes, observer);
if (bytesAfterMapping.isEmpty()) continue;
// todo 业务逻辑
//提交
kafkaConsumer.commitAsync(
(offsets, exception) -> {
if (exception != null) {
//报错,异常处理
monitor.reportAsync(MonitorType.COMMIT_OFFSET_FAILED, 1);
}
});
}
}
2.spring项目配置kafka
1.aplication.xml配置:
kafka:
topic: bi-sandbox
group-id: metadata-management-test
bootstrap-servers: 100.119.167.50:6189
enable-auto-commit: false
auto-offset-reset: earliest
session-timeout-ms: 20000
max-poll-interval-ms: 500000
max-poll-records: 2000
producer-topic: metadata-management-test
producer-bootstrap-servers: ckafka-3y356keb.ap-guangzhou.ckafka.tencentcloudmq.com:6001
2。使用ConfigurationProperties解析kafkaConfig配置文件:
@Component
@ConfigurationProperties(prefix = "kafka")
@Data
public class KafkaConfig {
private String topic;
private String groupId;
private String bootstrapServers;
private boolean enableAutoCommit;
private String autoOffsetReset;
private Integer sessionTimeoutMs;
private Integer maxPollIntervalMs;
private Integer maxPollRecords;
private String producerTopic;
private String producerBootstrapServers;
}
3.创建consumer properties:
/**
* 创建消费者
*
* @return
*/
public Properties consumerConfigs() {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs);
props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs);
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords);
//sasl配置
props.put("sasl.jaas.config",
"org.apache.kafka.common.security.plain.PlainLoginModule required "
+ "username=\"ckafka-kekvburn#action-data-rw\" password=\"XhzfvbErGLMus6TkHSBbN6Yt\";");
props.put("security.protocol", "SASL_PLAINTEXT");
props.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
return props;
}
4.创建实际的consumer消费者:
package com.tencent.metadata.api.kafka;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.Serdes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* kafka消费consumer
*
* @author bob
* @since 2020-10-26
*/
@Component
public class Consumer implements InitializingBean {
private static final Logger LOGGER = LoggerFactory.getLogger(Consumer.class);
@Resource
private com.tencent.metadata.api.config.KafkaConfig myConfig;
private static final Duration TIMEOUT = Duration.ofSeconds(10);
@Override
public void afterPropertiesSet() throws Exception {
new KafkaConsumerThread(myConfig).start();
}
public class KafkaConsumerThread extends Thread {
private final List<ActionMessage> buffer = new ArrayList<>(500);
private KafkaConsumer<byte[], byte[]> consumer;
public KafkaConsumerThread(com.tencent.metadata.api.config.KafkaConfig myConfig) {
this.consumer = provideKafkaClient(myConfig);
this.consumer.subscribe(Arrays.asList(myConfig.getTopic()));
}
private KafkaConsumer<byte[], byte[]> provideKafkaClient(
com.tencent.metadata.api.config.KafkaConfig myConfig) {
return new KafkaConsumer<>(
myConfig.consumerConfigs(), Serdes.ByteArray().deserializer(), Serdes.ByteArray().deserializer());
}
@Override
public void run() {
try {
while (!Thread.currentThread().isInterrupted()) {
ConsumerRecords<byte[], byte[]> records =
consumer.poll(TIMEOUT);
//解析数据
deserializeMessage(records);
if (buffer.isEmpty()) {
continue;
}
//有数据的时候的业务处理逻辑
//提交offset
consumer.commitAsync(
(offsets, exception) -> {
if (exception != null) {
LOGGER.warn("Offset commit failed: ", exception);
}
});
}
} catch (Exception e) {
LOGGER.error("consumer message error:", e);
} finally {
consumer.close();
}
}
/**
* 解析kafka消息
*
* @param records
*/
private void deserializeMessage(ConsumerRecords<byte[], byte[]> records) {
buffer.clear();
long serverReceiveTime = System.currentTimeMillis();
for (ConsumerRecord<byte[], byte[]> record : records) {
//解析消息
}
}
}
}
4.创建生产者properties:
/**
* 创建生产者
*
* @return
*/
public Properties producerConfigs() {
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, producerBootstrapServers);
props.put(ProducerConfig.ACKS_CONFIG, "all");
props.put(ProducerConfig.RETRIES_CONFIG, 0);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringSerializer");
props.put("sasl.jaas.config",
"org.apache.kafka.common.security.plain.PlainLoginModule required "
+ "username=\"ckafka-3y356keb#metadata-test\" password=\"XhzfvbErGLMus6TkHSBbN6Yt\";");
props.put("security.protocol", "SASL_PLAINTEXT");
props.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
return props;
}
4.创建实际的producer:
package com.tencent.metadata.api.kafka;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Random;
/**
* kafka生产producer
*
* @author bob
* @since 2020-11-16
*/
@Component
public class Producer implements InitializingBean {
private static final Logger LOGGER = LoggerFactory.getLogger(Producer.class);
@Resource
private com.tencent.metadata.api.config.KafkaConfig myConfig;
private static KafkaProducer<String, String> producer;
/**
* 初始化producer
* @throws Exception
*/
@Override
public void afterPropertiesSet() throws Exception {
producer = new KafkaProducer<String, String>(
myConfig.producerConfigs());
}
/**
* 发送消息
* @param data 消息体,json
*/
public void sendMessage(String data) {
ProducerRecord<String, String> message = new ProducerRecord<>(myConfig.getProducerTopic(),
new Random().nextInt(10) + "", data);
//异步发送
producer.send(message, new DemoProducerCallback());
}
/**
* 异步发送回调处理
*/
class DemoProducerCallback implements Callback {
@Override
public void onCompletion(RecordMetadata recordMetadata, Exception e) {
if (e != null) {
//如果Kafka返回一个错误,onCompletion方法抛出一个non null异常。
LOGGER.error("异步发送消息失败", e);
}
}
}
}