update
前言
最近在做对接 pulsarMQ 进行收发消息的任务。目的是在 A 服务中，生产消息并发送到 pulsarMQ，在 B 服务中消费消息，然后根据消息内容做一些业务处理。于是，我便对腾讯云的 tdmq (pulsar 版) 进行了学习。
对接步骤
用浏览器打开 腾讯云tdmq-pulsar
查看 tcp协议，java sdk。
按照官方文档，依次找到自己的 serviceUrl，token，topic （注意：tcp java sdk方式，topic需要以 persistent:// 开头）
代码
maven项目导入依赖
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId>
<version>2.10.0</version>
</dependency>
创建pulsarMQ配置类
@Configuration
public class PulsarMQConfig {
@Bean
@ConditionalOnMissingBean
public PulsarClient pulsarClient(Environment environment) throws PulsarClientException {
String serviceUrl = environment.getProperty("pulsar.service-url");
String token = environment.getProperty("pulsar.token-auth-value");
ClientBuilder pulsarClientBuilder = PulsarClient.builder()
// 服务接入地址
.serviceUrl(serviceUrl)
// 授权角色密钥
.authentication(AuthenticationFactory.token(token))
.ioThreads(5)
.listenerThreads(3)
.statsInterval(3, TimeUnit.SECONDS);
log.info("serviceUrl: {}", serviceUrl);
log.info(">> pulsar client created.");
return pulsarClientBuilder.build();
}
@Bean
@ConditionalOnProperty(value = "demoProducer.enable", havingValue = "1")
@DependsOn({"pulsarClient"})
public DemoProducer demoProducer(Environment environment, PulsarClient pulsarClient) {
return new DemoProducer(environment, pulsarClient);
}
@Bean
@ConditionalOnProperty(value = "demoConsumer.enable", havingValue = "1")
@DependsOn({"pulsarClient"})
public DemoConsumer demoConsumer(Environment environment, PulsarClient pulsarClient) {
return new DemoConsumer(environment, pulsarClient);
}
}
定义接口
生产者接口
public abstract class PulsarMQProducer<T> {
/**
* 1.发送消息的topic是在生产者配置中已经声明的topic
* 2.PulsarTemplate类型应于发送消息的类型一致
* 3.发送消息到指定topic时，消息类型需要与生产者工厂配置中的topic绑定的消息类型对应.
*
*/
protected Logger log;
/**
* 服务接入地址，位于【集群管理】页面接入地址
*/
protected String serviceUrl;
/**
* 要使用的命名空间授权的角密钥，位于【角色管理】页面
*/
protected String token;
protected String topic;
protected PulsarClient pulsarClient;
protected Producer<T> producer;
public void init(ProducerBuilder<T> producerBuilder) {
try {
log.info("topic: {}", topic);
producer = producerBuilder
.topic(topic)
.blockIfQueueFull(Boolean.TRUE)
.create();
log.info("create producer success");
} catch (PulsarClientException e) {
log.error("create producer error", e);
close();
}
}
public ProducerBuilder<byte[]> byteArraySchemaBuild() {
log.info("producer schemaBuild...");
ProducerBuilder<byte[]> producerBuilder = null;
try {
producerBuilder = pulsarClient.newProducer(Schema.BYTES);
} catch (Exception exception) {
log.error("producer schemaBuild exception, errorMsg: "+exception.getMessage(), exception);
close();
}
return producerBuilder;
}
public ProducerBuilder<T> jsonSchemaBuild(Class<T> clazz) {
log.info("producer schemaBuild...");
ProducerBuilder<T> producerBuilder = null;
try {
producerBuilder = pulsarClient.newProducer(Schema.JSON(clazz));
} catch (Exception exception) {
log.error("producer schemaBuild exception, errorMsg: "+exception.getMessage(), exception);
close();
}
return producerBuilder;
}
protected void asyncSendMessage(T msg, BiConsumer<? super MessageId, ? super Throwable> action) {
// 异步发送消息
// CompletableFuture<MessageId> completableFuture = producer.newMessage().value(msg).sendAsync();
// 通过异步回调得知消息发送成功与否
// completableFuture
// .whenComplete((messageId, throwable) -> func(msg, messageId, throwable))
// .whenComplete(action);
CompletableFuture.runAsync(() -> {
try {
// 生产者同步发送消息，等待broker返回发送结果
MessageId messageId = null;
messageId = producer.newMessage().value(msg).send();
action.accept(messageId, null);
} catch (Exception e) {
log.error("pulsar发送消息异常", e);
action.accept(null, e);
}
});
}
public void close() {
try {
if (Objects.nonNull(producer)) {
producer.close();
}
if (Objects.nonNull(pulsarClient)) {
pulsarClient.close();
}
log.info("close producer done.");
} catch (PulsarClientException exception) {
log.error("close producer error", exception);
}
}
}
消费者接口
public abstract class PulsarMQConsumer<T> {
/**
* 1.发送消息的topic是在生产者配置中已经声明的topic
* 2.PulsarTemplate类型应于发送消息的类型一致
* 3.发送消息到指定topic时，消息类型需要与生产者工厂配置中的topic绑定的消息类型对应.
*/
protected Logger logger;
/**
* 服务接入地址，位于【集群管理】页面接入地址
*/
protected String serviceUrl;
/**
* 要使用的命名空间授权的角密钥，位于【角色管理】页面
*/
protected String token;
protected String topic;
protected String subscribeName;
protected PulsarClient pulsarClient;
protected Consumer<T> consumer;
protected ExecutorService executor;
public void init(ConsumerBuilder<T> consumerBuilder) {
try {
logger.info("topic: {},subscribeName: {}", topic, subscribeName);
consumer = consumerBuilder
.topic(topic)
.subscriptionName(subscribeName)
// 15分钟没ack，重传
.ackTimeout(15, TimeUnit.MINUTES)
// 被累积在消费者内存的消息数，调小该值，可以降低broker unack消息数
// 避免 unack 达到5000，触发 tdmq pulsar的虚拟集群的最大已接收未确认消息数量限制
// 导致大量消费滞留在broker侧无法消费
// https://pulsar.apache.org/docs/2.11.x/client-libraries-java/#configure-consumer
// https://cloud.tencent.com/document/product/1179/57399#.E6.B6.88.E6.81.AF
.receiverQueueSize(200)
// 共享模式
.subscriptionType(SubscriptionType.Shared)
.enableRetry(true)
.deadLetterPolicy(DeadLetterPolicy.builder()
.maxRedeliverCount(5)
.build())
// 配置从最早开始消费，否则可能会消费不到历史消息
.subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
.messageListener((consumer, message) -> {
executor.execute(() -> {
outerProcess(message, consumer);
});
})
.subscribe();
logger.info("create consumer success");
} catch (PulsarClientException exception) {
logger.error("create consumer error", exception);
close();
}
}
public void outerProcess(Message<T> message, Consumer<T> consumer) {
try {
process(message);
consumer.acknowledge(message);
} catch (PulsarClientException exception) {
logger.error("发生 pulsar client 异常, messageId: {}, data: {}",
message.getMessageId(), message.getValue());
logger.error("pulsar client error, messageId: " + message.getMessageId(), exception);
occurExceptionReSendMsg(message, consumer);
} catch (BusinessException exception) {
logger.error("发生业务异常, messageId: {}, data: {}",
message.getMessageId(), message.getValue());
logger.error("业务异常, messageId: " + message.getMessageId(), exception);
occurExceptionReSendMsg(message, consumer);
} catch (Exception exception) {
logger.error("发生未知异常, messageId: {}, data: {}",
message.getMessageId(), message.getValue());
logger.error("发生未知异常,messageId: " + message.getMessageId(), exception);
occurExceptionReSendMsg(message, consumer);
}
}
void occurExceptionReSendMsg(Message<T> message, Consumer<T> consumer) {
try {
String reconsumetimes = message.getProperty("RECONSUMETIMES");
int cnt = 1;
if (reconsumetimes != null) {
cnt = Integer.parseInt(reconsumetimes);
}
consumer.reconsumeLater(message, cnt << 1, TimeUnit.MINUTES);
} catch (PulsarClientException exception) {
logger.error("reconsumer message error, messageId: " + message.getMessageId(), exception);
}
}
public ConsumerBuilder<byte[]> byteArraySchemaBuild() {
logger.info("consumer schemaBuild...");
ConsumerBuilder<byte[]> consumerBuilder = null;
try {
consumerBuilder = pulsarClient.newConsumer(Schema.BYTES);
} catch (Exception exception) {
logger.error("consumer schemaBuild exception, errorMsg: "+exception.getMessage(), exception);
close();
}
return consumerBuilder;
}
public ConsumerBuilder<T> jsonSchemaBuild(Class<T> clazz) {
logger.info("consumer schemaBuild...");
ConsumerBuilder<T> consumerBuilder = null;
try {
consumerBuilder = pulsarClient.newConsumer(Schema.JSON(clazz));
} catch (Exception exception) {
logger.error("consumer schemaBuild exception, errorMsg: "+exception.getMessage(), exception);
close();
}
return consumerBuilder;
}
public abstract void process(Message<T> message) throws PulsarClientException;
public void close() {
try {
if (executor != null) {
executor.shutdownNow();
}
if (Objects.nonNull(consumer)) {
consumer.close();
}
if (pulsarClient != null) {
pulsarClient.close();
}
logger.info("close consumer done.");
} catch (PulsarClientException exception) {
logger.error("close consumer error", exception);
}
}
}
创建生产者
public class DemoProducer extends PulsarMQProducer<DemoMessage> implements DisposableBean {
public IntelMarkMsgProducer(Environment environment, PulsarClient pulsarClient) {
log = LoggerFactory.getLogger(DemoProducer.class);
serviceUrl = environment.getProperty("service-url");
token = environment.getProperty("token-auth-value");
topic = environment.getProperty("topic");
this.pulsarClient = pulsarClient;
super.init(jsonSchemaBuild(DemoMessage.class));
}
@Override
public void destroy() throws Exception {
super.close();
}
}
创建消费者
public class DemoConsumer extends PulsarMQConsumer<DemoMessage> implements DisposableBean {
public IntelMarkMsgConsumer(Environment environment, PulsarClient pulsarClient) {
logger = LoggerFactory.getLogger(DemoConsumer.class);
serviceUrl = environment.getProperty("service-url");
token = environment.getProperty("token-auth-value");
topic = environment.getProperty("topic");
subscribeName = environment.getProperty("subscribeName");
executor = new ThreadPoolExecutor(
2,
5,
60,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
new ThreadFactoryBuilder().setNameFormat("DemoConsumer-thread-pool-%d").build(),
new ThreadPoolExecutor.CallerRunsPolicy()
);
this.pulsarClient = pulsarClient;
super.init(jsonSchemaBuild(DemoMessage.class));
}
@Override
public void process(Message<DemoMessage> message) {
// do something
}
@Override
public void destroy() throws Exception {
super.close();
}
}
3934
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
