整理的springboot整合Pulsar的案例 (TDMQ)

前言:

由于时间比较赶,TDMQ被项目启用了,很多高阶的还没有用上,这里只是简单的发送接收

初始化连接

package com.menglar.soap.item.common.pulsar;

import org.apache.pulsar.client.api.AuthenticationFactory;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.shade.io.netty.util.concurrent.DefaultThreadFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import static java.util.concurrent.Executors.newScheduledThreadPool;

/**
 *•ioThreads netty 的 ioThreads 负责网络 IO 操作，如果业务流量较大，可以调高ioThreads个数；
 * •listenersThreads 负责调用以listener模式启动的消费者的回调函数，建议配置大于该 client 负责的partition数目；
 * •memoryLimit 当前用于限制pulsar生产者可用的最大内存，可以很好地防止网络中断、Pulsar 故障等场景下，消息积压在producer侧，导致 Java 程序 OOM；
 * •operationTimeout 一些元数据操作的超时时间，Pulsar 默认为 30s，有些保守，可以根据自己的网络情况、处理性能来适当调低；
 * •connectionTimeout 连接 Pulsar 的超时时间，配置原则同上。
 */

@Component
public class PulsarClientInit {
    @Value("${pulsar.token}")
    private String token;
	  @Value("${pulsar.service-url}")
   	private String serviceUrl;
    private  PulsarClient pulsars;

    private final ScheduledExecutorService executorService =
           newScheduledThreadPool(1, new DefaultThreadFactory("pulsar-cli-init"));

    public void init() {
        executorService.scheduleWithFixedDelay(this::initWithRetry, 0, 10, TimeUnit.SECONDS);
    }

    @PostConstruct
    private void initWithRetry() {
        try {
            PulsarClient pulsarClient = PulsarClient.builder()
                    .serviceUrl(serviceUrl)
                    .ioThreads(4)
                    .listenerThreads(10)
                    .operationTimeout(5, TimeUnit.SECONDS)
                    .connectionTimeout(15, TimeUnit.SECONDS)
                    .authentication(AuthenticationFactory.token(token))
                    .build();
            System.out.println("pulsar初始化client成功");
           this.setPulsars(pulsarClient);
            this.executorService.shutdown();
        } catch (Exception e) {
            System.out.println("pulsar初始化client异常, exception is "+e);
        }
    }

    public PulsarClient getPulsars() {
        return pulsars;
    }

    public void setPulsars(PulsarClient pulsars) {
        this.pulsars = pulsars;
    }
}

生产者

package com.menglar.soap.item.common.pulsar;

import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.nacos.common.utils.JacksonUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;

@Slf4j
@Component
public class PulsarProducer {
	static PulsarClient client = null;
	static Producer<byte[]> producer = null;

	@Value("${pulsar.serviceUrl}")
	private final static String serviceUrl = "http://pulsar-rkrw3x49gxao.tdmq.ap-gz.public.tencenttdmq.com:8080";
	@Value("${pulsar.token}")
	private final static String token = "eyJrZXlJZCI6InB1bHNhci1ya3J3M3g0OWd4YW8iLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItcmtydzN4NDlneGFvX2FkbWluIn0.8j1ETGbGDLy0rd3xxFNsDxLtQWm2vy8ScgyLM0hEFa8";
	@Value("${pulsar.topic}")
	private final static String topic = "pulsar-rkrw3x49gxao/demo_pulsar/mengla";

	@PostConstruct
	public void initPulsar() throws PulsarClientException {
		//构造Pulsar client
		client = PulsarClient.builder()
			.serviceUrl(serviceUrl)
			.authentication(AuthenticationFactory.token(token))
			.build();
	}

	/**
	 * 发送消息
	 *
	 * @param key  key
	 * @param data data
	 * @return MessageId
	 * @since 2021/5/6 16:46
	 */
	public MessageId sendMsg(String key, String data) {
		CompletableFuture<MessageId> future = producer.newMessage()
			.key(key)
			//异步发送
			.value(data.getBytes()).sendAsync();
		future.handle((v, ex) -> {
			if (ex == null) {
				log.info("Message persisted, MessageId:{}, data:{}", v, data);
			} else {
				log.error("发送消息失败msg:{} ", data, ex);
			}
			return null;
		});
		return future.join();
	}


	/**
	 * 发送消息带延迟时间
	 *
	 * @param topic    主题
	 * @param message  消息
	 * @param catchEx  是否获取异常
	 * @param delay    延迟时间
	 * @param timeUnit 时间单位
	 * @param <T>      泛型
	 * @return
	 */
	public static <T> String send(String topic, T message, boolean catchEx, Long delay, TimeUnit timeUnit) throws PulsarClientException {
		build(topic);

		TypedMessageBuilder<byte[]> builder = producer.newMessage();
		builder.value(JacksonUtils.toJsonBytes(message));
		if (ObjectUtil.isAllNotEmpty(delay, timeUnit)) {
			builder.deliverAfter(delay, timeUnit);
		}
		String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
		try {
			log.info("发送MQ消息topic:{}，message:{}", topic, msg);

			return builder.send().toString();

		} catch (Exception e) {
			log.error("发送topic:{}消息失败，message:{}", topic, msg);
			if (!catchEx) {
				throw new RuntimeException("mq发送失败");
			}
			return null;
		}
	}

	private static void build(String topic) throws PulsarClientException {
		producer = client.newProducer()
			.topic(topic)
			.enableBatching(true)
			.compressionType(CompressionType.LZ4)
			.batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
			.sendTimeout(0, TimeUnit.SECONDS)
			.batchingMaxMessages(1000)
			.maxPendingMessages(1000)
			.blockIfQueueFull(true)
			.roundRobinRouterBatchingPartitionSwitchFrequency(10)
			.batcherBuilder(BatcherBuilder.DEFAULT)
			.create();
	}

	/**
	 * 发送普通消息
	 *
	 * @param topic
	 * @param message
	 * @param <T>
	 * @return
	 * @throws PulsarClientException
	 */
	public static  <T> String send(String topic, T message) throws PulsarClientException {
		build(topic);
		TypedMessageBuilder<byte[]> builder = producer.newMessage();
		builder.value(JacksonUtils.toJsonBytes(message));

		String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
		try {
			log.info("发送MQ消息topic:{}，message:{}", topic, msg);

			return builder.send().toString();

		} catch (Exception e) {
			log.error("发送topic:{}消息失败，message:{}", topic, msg);
			return null;
		}
	}
}

消费者

package com.menglar.soap.item.common.pulsar;

import com.alibaba.druid.sql.visitor.SQLASTOutputVisitor;
import org.apache.pulsar.client.api.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.util.concurrent.TimeUnit;

@Component
public class PulsarConsumer {
    private final static String TOPIC = "pulsar-rkrw3x49gxao/demo_pulsar/mengla";
    private final static String TOPIC1 = "pulsar-rkrw3x49gxao/demo_pulsar/first_demo";
    private final static String SUBSCRIPTION = "dddd";

	@Value("${pulsar.serviceUrl}")
	private final static String serviceUrl = "http://pulsar-rkrw3x49gxao.tdmq.ap-gz.public.tencenttdmq.com:8080";
	@Value("${pulsar.token}")
	private final static String token = "eyJrZXlJZCI6InB1bHNhci1ya3J3M3g0OWd4YW8iLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItcmtydzN4NDlneGFvX2FkbWluIn0.8j1ETGbGDLy0rd3xxFNsDxLtQWm2vy8ScgyLM0hEFa8";

	static PulsarClientInit pulsarclientinitultimate =null;
	public  PulsarConsumer(PulsarClientInit pulsarclientinitultimate) {
		PulsarConsumer.pulsarclientinitultimate = pulsarclientinitultimate;
	}

//	@PostConstruct
//	public void initPulsar() throws PulsarClientException {
//		//构造Pulsar client
//		client = PulsarClient.builder()
//			.serviceUrl(serviceUrl)
//			.authentication(AuthenticationFactory.token(token))
//			.build();
//	}


	@PostConstruct
   @SuppressWarnings("unchecked")
    public void init(){
		PulsarClient client = pulsarclientinitultimate.getPulsars();
		try {
			Consumer<byte[]> subscribe = client.newConsumer()
				.subscriptionName(SUBSCRIPTION)
				.topic(TOPIC).
					subscriptionType(SubscriptionType.Shared)
				.messageListener((MessageListener<byte[]>) (consumer, message) -> {
					System.out.println("consumer = " + message.getTopicName());
					System.out.println("consumer = " +  message.getPublishTime());
				})
				.subscribe();
			Message<byte[]> receive = subscribe.receive();
			MessageId messageId = receive.getMessageId();
			System.out.println("messageId = " + messageId);
			byte[] data = receive.getData();
			System.out.println("data = " + data);

		} catch (PulsarClientException e) {
			e.printStackTrace();
		}
	}
//    @PostConstruct
//    @SuppressWarnings("unchecked")
//    public void init() throws PulsarClientException {
//			PulsarClient client = pulsarclientinitultimate.getPulsars();
//			Consumer m = client.newConsumer()
//				.topic(TOPIC)
//				.subscriptionName(SUBSCRIPTION)
//				.receiverQueueSize(100)
//				.ackTimeout(1, TimeUnit.SECONDS)
//				.subscriptionType(SubscriptionType.Key_Shared)
//				.negativeAckRedeliveryDelay(1, TimeUnit.SECONDS)
                .deadLetterPolicy(
                        DeadLetterPolicy.builder()
                        //可以指定最大重试次数，最大重试三次后，进入到死信队列
                        .maxRedeliverCount(3)
//				//可以指定死信队列
//				//.deadLetterTopic("pulsar-wb2ve9rnv8on/books/users")
                        .build())
//				.messageListener((MessageListener) (consumer, msg) -> {
//
//					System.out.println("msg.getValue().toString() = " + msg.getValue());
//
//					System.out.println("接收到队列「{}」消息:{} 主题名是" + msg.getTopicName() + "-----" + msg.getValue());
//					if (msg.getValue().equals("m")) {
//						throw new RuntimeException("hello3消息消费失败！");
//					} else {
//						try {
//							consumer.acknowledge(msg);
//						} catch (PulsarClientException e) {
//							e.printStackTrace();
//						}
//					}
//				}).subscribe();
//			Message receive = m.receive();
//			String s = receive.getData().toString();
//			System.out.println("s = " + s);
//		}
}

工具类(自己看着用)

package com.zrt.demo.controller;

import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSONObject;
import com.menglar.soap.item.common.pulsar.PulsarClientInit;
import org.apache.pulsar.client.api.*;
import org.springframework.stereotype.Component;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;

/**
 * @Author: zqf
 * @Date: 2021/12/05/16:06
 * @Description: pulsar发送消息的工具类
 */
@Component
public class PulsarUtils {

    static PulsarClientInit pulsarclientinitultimate =null;

    public  PulsarUtils(PulsarClientInit pulsarclientinitultimate) {
        PulsarUtils.pulsarclientinitultimate = pulsarclientinitultimate;
    }

    /**
     * 初始化Producer
     * @param topic 主题
     * @return producer
     * @throws PulsarClientException
     */
    private static TypedMessageBuilder  initProducer(String topic) throws PulsarClientException {
        PulsarClient client = pulsarclientinitultimate.getPulsars();

        return  client.newProducer()
                .topic(topic)
                .create()
                .newMessage(Schema.BYTES);
    }

    /**
     * 异步发送
     * @param topic 主题t
     * @param message 要发送的数据
     */
    public static <T> void sendAnsyMsg(String topic,  T message) throws PulsarClientException {
        TypedMessageBuilder builder = initProducer(topic);
        CompletableFuture future = builder.value(JSONObject.toJSONBytes(message)).sendAsync();//异步发送
        future.handle((v, ex) -> {
            if (ex == null) {
                System.out.println("Message persisted2: {}"+message);
            } else {
                System.out.println("发送Pulsar消息失败msg:【{}】 "+message+ex);
            }
            return null;
        });
        // future.join();
    }

    /**
     * 延迟发送消息
     * @param topic    主题
     * @param message  消息
     * @param catchEx  是否捕获异常
     * @param delay    延迟时间
     * @param timeUnit 单位
     * @param <T>      消息类型
     * @return mq消息ID
     */
    public static <T> String send(String topic, T message, boolean catchEx, Long delay, TimeUnit timeUnit) throws PulsarClientException {
        TypedMessageBuilder<byte[]> builder = initProducer(topic);
        if (ObjectUtil.isAllNotEmpty(delay, timeUnit)) {
            builder.deliverAfter(delay, timeUnit);
        }
        String msg = message instanceof Number ? message.toString() : JSONUtil.toJsonStr(message);
        try {
            System.out.println("发送MQ消息topic:{}，message:{}"+ topic+ "---"+msg);
            return builder.send().toString();
        } catch (Exception e) {
            System.out.println("发送topic:{}消息失败，message:{}"+topic+msg);
            if (!catchEx) {
                throw new RuntimeException("mq发送失败");
            }
            return null;
        }
    }

    /**
     * 简单发送需要捕获异常
     * @param topic 主题
     * @param message 信息
     * @param catchEx 是否捕获异常
     * @param <T> 消息类型
     * @return
     * @throws PulsarClientException
     */
    public static <T> String send(String topic, T message, boolean catchEx) throws PulsarClientException {
        TypedMessageBuilder<byte[]> builder = initProducer(topic);
        return send(topic, message, catchEx, null, null);
    }

    /**
     * 延迟发送消息
     * @param topic    主题
     * @param message  消息
     * @param delay    延迟时间
     * @param timeUnit 单位
     * @param <T>      消息类型
     * @return mq消息ID
     */

    public static <T> String send(String topic, T message, Long delay, TimeUnit timeUnit) throws PulsarClientException {
        return send(topic, message, true, delay, timeUnit);
    }

    /**
     * 简单发送
     * @param topic 主题
     * @param message 消息
     * @param <T> 类型
     * @return
     * @throws PulsarClientException
     */
    public static <T> String send(String topic, T message) throws PulsarClientException {
        return send(topic, message, true,null,null);
    }
}