前言:
由于时间比较赶,TDMQ被项目启用了,很多高阶的还没有用上,这里只是简单的发送接收
初始化连接
package com.menglar.soap.item.common.pulsar;
import org.apache.pulsar.client.api.AuthenticationFactory;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.shade.io.netty.util.concurrent.DefaultThreadFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static java.util.concurrent.Executors.newScheduledThreadPool;
/**
*•ioThreads netty 的 ioThreads 负责网络 IO 操作,如果业务流量较大,可以调高ioThreads个数;
* •listenersThreads 负责调用以listener模式启动的消费者的回调函数,建议配置大于该 client 负责的partition数目;
* •memoryLimit 当前用于限制pulsar生产者可用的最大内存,可以很好地防止网络中断、Pulsar 故障等场景下,消息积压在producer侧,导致 Java 程序 OOM;
* •operationTimeout 一些元数据操作的超时时间,Pulsar 默认为 30s,有些保守,可以根据自己的网络情况、处理性能来适当调低;
* •connectionTimeout 连接 Pulsar 的超时时间,配置原则同上。
*/
@Component
public class PulsarClientInit {
@Value("${pulsar.token}")
private String token;
@Value("${pulsar.service-url}")
private String serviceUrl;
private PulsarClient pulsars;
private final ScheduledExecutorService executorService =
newScheduledThreadPool(1, new DefaultThreadFactory("pulsar-cli-init"));
public void init() {
executorService.scheduleWithFixedDelay(this::initWithRetry, 0, 10, TimeUnit.SECONDS);
}
@PostConstruct
private void initWithRetry() {
try {
PulsarClient pulsarClient = PulsarClient.builder()
.serviceUrl(serviceUrl)
.ioThreads(4)
.listenerThreads(10)
.operationTimeout(5, TimeUnit.SECONDS)
.connectionTimeout(15, TimeUnit.SECONDS)
.authentication(AuthenticationFactory.token(token))
.build();
System.out.println("pulsar初始化client成功");
this.setPulsars(pulsarClient);
this.executorService.shutdown();
} catch (Exception e) {
System.out.println("pulsar初始化client异常, exception is "+e);
}
}
public PulsarClient getPulsars() {
return pulsars;
}
public void setPulsars(PulsarClient pulsars) {
this.pulsars = pulsars;
}
}
生产者
package com.menglar.soap.item.common.pulsar;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.nacos.common.utils.JacksonUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
public class PulsarProducer {
static PulsarClient client = null;
static Producer<byte[]> producer = null;
@Value("${pulsar.serviceUrl}")
private final static String serviceUrl = "http://pulsar-rkrw3x49gxao.tdmq.ap-gz.public.tencenttdmq.com:8080";
@Value("${pulsar.token}")
private final static String token = "eyJrZXlJZCI6InB1bHNhci1ya3J3M3g0OWd4YW8iLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItcmtydzN4NDlneGFvX2FkbWluIn0.8j1ETGbGDLy0rd3xxFNsDxLtQWm2vy8ScgyLM0hEFa8";
@Value("${pulsar.topic}")
private final static String topic = "pulsar-rkrw3x49gxao/demo_pulsar/mengla";
@PostConstruct
public void initPulsar() throws PulsarClientException {
//构造Pulsar client
client = PulsarClient.builder()
.serviceUrl(serviceUrl)
.authentication(AuthenticationFactory.token(token))
.build();
}
/**
* 发送消息
*
* @param key key
* @param data data
* @return MessageId
* @since 2021/5/6 16:46
*/
public MessageId sendMsg(String key, String data) {
CompletableFuture<MessageId> future = producer.newMessage()
.key(key)
//异步发送
.value(data.getBytes()).sendAsync();
future.handle((v, ex) -> {
if (ex == null) {
log.info("Message persisted, MessageId:{}, data:{}", v, data);
} else {
log.error("发送消息失败msg:{} ", data, ex);
}
return null;
});
return future.join();
}
/**
* 发送消息带延迟时间
*
* @param topic 主题
* @param message 消息
* @param catchEx 是否获取异常
* @param delay 延迟时间
* @param timeUnit 时间单位
* @param <T> 泛型
* @return
*/
public static <T> String send(String topic, T message, boolean catchEx, Long delay, TimeUnit timeUnit) throws PulsarClientException {
build(topic);
TypedMessageBuilder<byte[]> builder = producer.newMessage();
builder.value(JacksonUtils.toJsonBytes(message));
if (ObjectUtil.isAllNotEmpty(delay, timeUnit)) {
builder.deliverAfter(delay, timeUnit);
}
String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
try {
log.info("发送MQ消息topic:{},message:{}", topic, msg);
return builder.send().toString();
} catch (Exception e) {
log.error("发送topic:{}消息失败,message:{}", topic, msg);
if (!catchEx) {
throw new RuntimeException("mq发送失败");
}
return null;
}
}
private static void build(String topic) throws PulsarClientException {
producer = client.newProducer()
.topic(topic)
.enableBatching(true)
.compressionType(CompressionType.LZ4)
.batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
.sendTimeout(0, TimeUnit.SECONDS)
.batchingMaxMessages(1000)
.maxPendingMessages(1000)
.blockIfQueueFull(true)
.roundRobinRouterBatchingPartitionSwitchFrequency(10)
.batcherBuilder(BatcherBuilder.DEFAULT)
.create();
}
/**
* 发送普通消息
*
* @param topic
* @param message
* @param <T>
* @return
* @throws PulsarClientException
*/
public static <T> String send(String topic, T message) throws PulsarClientException {
build(topic);
TypedMessageBuilder<byte[]> builder = producer.newMessage();
builder.value(JacksonUtils.toJsonBytes(message));
String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
try {
log.info("发送MQ消息topic:{},message:{}", topic, msg);
return builder.send().toString();
} catch (Exception e) {
log.error("发送topic:{}消息失败,message:{}", topic, msg);
return null;
}
}
}
消费者
package com.menglar.soap.item.common.pulsar;
import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import org.apache.pulsar.client.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.concurrent.TimeUnit;
@Component
public class PulsarConsumer {
private final static String TOPIC = "pulsar-rkrw3x49gxao/demo_pulsar/mengla";
private final static String TOPIC1 = "pulsar-rkrw3x49gxao/demo_pulsar/first_demo";
private final static String SUBSCRIPTION = "dddd";
@Value("${pulsar.serviceUrl}")
private final static String serviceUrl = "http://pulsar-rkrw3x49gxao.tdmq.ap-gz.public.tencenttdmq.com:8080";
@Value("${pulsar.token}")
private final static String token = "eyJrZXlJZCI6InB1bHNhci1ya3J3M3g0OWd4YW8iLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItcmtydzN4NDlneGFvX2FkbWluIn0.8j1ETGbGDLy0rd3xxFNsDxLtQWm2vy8ScgyLM0hEFa8";
static PulsarClientInit pulsarclientinitultimate =null;
public PulsarConsumer(PulsarClientInit pulsarclientinitultimate) {
PulsarConsumer.pulsarclientinitultimate = pulsarclientinitultimate;
}
// @PostConstruct
// public void initPulsar() throws PulsarClientException {
// //构造Pulsar client
// client = PulsarClient.builder()
// .serviceUrl(serviceUrl)
// .authentication(AuthenticationFactory.token(token))
// .build();
// }
@PostConstruct
@SuppressWarnings("unchecked")
public void init(){
PulsarClient client = pulsarclientinitultimate.getPulsars();
try {
Consumer<byte[]> subscribe = client.newConsumer()
.subscriptionName(SUBSCRIPTION)
.topic(TOPIC).
subscriptionType(SubscriptionType.Shared)
.messageListener((MessageListener<byte[]>) (consumer, message) -> {
System.out.println("consumer = " + message.getTopicName());
System.out.println("consumer = " + message.getPublishTime());
})
.subscribe();
Message<byte[]> receive = subscribe.receive();
MessageId messageId = receive.getMessageId();
System.out.println("messageId = " + messageId);
byte[] data = receive.getData();
System.out.println("data = " + data);
} catch (PulsarClientException e) {
e.printStackTrace();
}
}
// @PostConstruct
// @SuppressWarnings("unchecked")
// public void init() throws PulsarClientException {
// PulsarClient client = pulsarclientinitultimate.getPulsars();
// Consumer m = client.newConsumer()
// .topic(TOPIC)
// .subscriptionName(SUBSCRIPTION)
// .receiverQueueSize(100)
// .ackTimeout(1, TimeUnit.SECONDS)
// .subscriptionType(SubscriptionType.Key_Shared)
// .negativeAckRedeliveryDelay(1, TimeUnit.SECONDS)
.deadLetterPolicy(
DeadLetterPolicy.builder()
//可以指定最大重试次数,最大重试三次后,进入到死信队列
.maxRedeliverCount(3)
// //可以指定死信队列
// //.deadLetterTopic("pulsar-wb2ve9rnv8on/books/users")
.build())
// .messageListener((MessageListener) (consumer, msg) -> {
//
// System.out.println("msg.getValue().toString() = " + msg.getValue());
//
// System.out.println("接收到队列「{}」消息:{} 主题名是" + msg.getTopicName() + "-----" + msg.getValue());
// if (msg.getValue().equals("m")) {
// throw new RuntimeException("hello3消息消费失败!");
// } else {
// try {
// consumer.acknowledge(msg);
// } catch (PulsarClientException e) {
// e.printStackTrace();
// }
// }
// }).subscribe();
// Message receive = m.receive();
// String s = receive.getData().toString();
// System.out.println("s = " + s);
// }
}
工具类(自己看着用)
package com.zrt.demo.controller;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSONObject;
import com.menglar.soap.item.common.pulsar.PulsarClientInit;
import org.apache.pulsar.client.api.*;
import org.springframework.stereotype.Component;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
/**
* @Author: zqf
* @Date: 2021/12/05/16:06
* @Description: pulsar发送消息的工具类
*/
@Component
public class PulsarUtils {
static PulsarClientInit pulsarclientinitultimate =null;
public PulsarUtils(PulsarClientInit pulsarclientinitultimate) {
PulsarUtils.pulsarclientinitultimate = pulsarclientinitultimate;
}
/**
* 初始化Producer
* @param topic 主题
* @return producer
* @throws PulsarClientException
*/
private static TypedMessageBuilder initProducer(String topic) throws PulsarClientException {
PulsarClient client = pulsarclientinitultimate.getPulsars();
return client.newProducer()
.topic(topic)
.create()
.newMessage(Schema.BYTES);
}
/**
* 异步发送
* @param topic 主题t
* @param message 要发送的数据
*/
public static <T> void sendAnsyMsg(String topic, T message) throws PulsarClientException {
TypedMessageBuilder builder = initProducer(topic);
CompletableFuture future = builder.value(JSONObject.toJSONBytes(message)).sendAsync();//异步发送
future.handle((v, ex) -> {
if (ex == null) {
System.out.println("Message persisted2: {}"+message);
} else {
System.out.println("发送Pulsar消息失败msg:【{}】 "+message+ex);
}
return null;
});
// future.join();
}
/**
* 延迟发送消息
* @param topic 主题
* @param message 消息
* @param catchEx 是否捕获异常
* @param delay 延迟时间
* @param timeUnit 单位
* @param <T> 消息类型
* @return mq消息ID
*/
public static <T> String send(String topic, T message, boolean catchEx, Long delay, TimeUnit timeUnit) throws PulsarClientException {
TypedMessageBuilder<byte[]> builder = initProducer(topic);
if (ObjectUtil.isAllNotEmpty(delay, timeUnit)) {
builder.deliverAfter(delay, timeUnit);
}
String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
try {
System.out.println("发送MQ消息topic:{},message:{}"+ topic+ "---"+msg);
return builder.send().toString();
} catch (Exception e) {
System.out.println("发送topic:{}消息失败,message:{}"+topic+msg);
if (!catchEx) {
throw new RuntimeException("mq发送失败");
}
return null;
}
}
/**
* 简单发送需要捕获异常
* @param topic 主题
* @param message 信息
* @param catchEx 是否捕获异常
* @param <T> 消息类型
* @return
* @throws PulsarClientException
*/
public static <T> String send(String topic, T message, boolean catchEx) throws PulsarClientException {
TypedMessageBuilder<byte[]> builder = initProducer(topic);
return send(topic, message, catchEx, null, null);
}
/**
* 延迟发送消息
* @param topic 主题
* @param message 消息
* @param delay 延迟时间
* @param timeUnit 单位
* @param <T> 消息类型
* @return mq消息ID
*/
public static <T> String send(String topic, T message, Long delay, TimeUnit timeUnit) throws PulsarClientException {
return send(topic, message, true, delay, timeUnit);
}
/**
* 简单发送
* @param topic 主题
* @param message 消息
* @param <T> 类型
* @return
* @throws PulsarClientException
*/
public static <T> String send(String topic, T message) throws PulsarClientException {
return send(topic, message, true,null,null);
}
}