中间件(3) : rocketmq消费历史消息

参考 : JAVA代码实现RocketMQ消息发送和接收_陈家宝的博客-CSDN博客_java rocketmq

 说明

消费全部tag为

defaultMQPushConsumer.subscribe("jack-topic", "*")

 指定tag为(多个tag用两个竖杆分隔)

defaultMQPushConsumer.subscribe("jack-topic", "t1")

消费起始时间设置(时间格式为yyyyMMddHHmmss)

            defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);
defaultMQPushConsumer.setConsumeTimestamp("20210929102009");

maven

        <dependency>
            <groupId>org.apache.rocketmq</groupId>
            <artifactId>rocketmq-spring-boot-starter</artifactId>
            <version>2.1.1</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.58</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.14</version>
            <scope>provided</scope>
        </dependency>

注意线程安全

 1.测试版

package cn.jack.mq;
 
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.common.message.MessageExt;
 
import java.util.List;
 
public class RocketMQReceiveMessageTest {
    public static void main(String[] args) throws Exception {
        // TODO 1 创建消费者，指定所属的消费者组名
        DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer("jack-consumer-group");
        // TODO 2 指定NameServer的地址
        defaultMQPushConsumer.setNamesrvAddr("192.168.232.129:9876");
        // TODO 3 指定消费者订阅的主题和标签
        defaultMQPushConsumer.subscribe("jack-topic", "*");
        // TODO 4 进行订阅：注册回调函数，编写处理消息的逻辑
        defaultMQPushConsumer.registerMessageListener((List<MessageExt> list, ConsumeConcurrentlyContext context) -> {
 
            // 1 try catch(throwable)确保不会因为业务逻辑的异常，导致消息出现重复消费的现象
            // 2 org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService.ConsumeRequest.run()中会对Throwable进行捕获，
            //   并且返回ConsumeConcurrentlyStatus.RECONSUME_LATER
            try {
                System.out.println("收到消息--》" + list);
                // 模拟业务异常
                int i = 1 / 0;
            } catch (Throwable throwable) {
                throwable.printStackTrace();
            }
 
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        });
 
        // TODO 5 启动消费者
        defaultMQPushConsumer.start();
        System.out.println("消费者启动成功。。。");
    }
}

2.springboot项目应用

import com.alibaba.fastjson.JSON;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageExt;
import org.springframework.stereotype.Service;

import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;

/**
 * @Author: liyue
 * @Date: 2021/09/29/10:31
 * @Description:
 */
@Service
@Slf4j
public class ConsumerService {

    private Map<String, ConcurrentLinkedQueue<MsgDTO>> msgs = new ConcurrentHashMap<>();

    private Map<String, Boolean> consumerStatus = new ConcurrentHashMap<>();

    private DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer("linshi-qa-consumer-" + UUID.randomUUID());


    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    public static class MsgDTO {
        private Long time;
        private String other;
    }

    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    public static class ConsumerParamTDO {

        private Long startTime;
        private Long endTime;
        private String subExpression;
        private String namesrvAddr;
        private String topic;
    }

    public ConcurrentLinkedQueue<MsgDTO> consumer(ConsumerParamTDO consumerParamTDO) {
        try {
            String uuid = UUID.randomUUID().toString();
            // 指定NameServer的地址
            defaultMQPushConsumer.setNamesrvAddr(consumerParamTDO.getNamesrvAddr());
            // 指定消费者订阅的主题和标签
            defaultMQPushConsumer.subscribe(consumerParamTDO.getTopic(), consumerParamTDO.getSubExpression());
            defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);
            defaultMQPushConsumer.setConsumeTimestamp(getSdf("yyyyMMddHHmmss").format(new Date(consumerParamTDO.getStartTime())));
            // 进行订阅：注册回调函数，编写处理消息的逻辑
            defaultMQPushConsumer.registerMessageListener((List<MessageExt> list, ConsumeConcurrentlyContext context) -> {
                for (MessageExt messageExt : list) {
                    String msg = new String(messageExt.getBody());
                    MsgDTO msgDTO = JSON.parseObject(msg, MsgDTO.class);
                    if (msgs.containsKey(uuid)) {
                        msgs.get(uuid).offer(msgDTO);
                    } else {
                        ConcurrentLinkedQueue<MsgDTO> msgDTOS = new ConcurrentLinkedQueue<>();
                        msgDTOS.offer(msgDTO);
                        msgs.put(uuid, msgDTOS);
                    }
                    if (msgDTO.getTime() > consumerParamTDO.getEndTime()) {
                        consumerStatus.put(uuid, Boolean.TRUE);
                        defaultMQPushConsumer.shutdown();
                    }
                }
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            });
            try {
                defaultMQPushConsumer.start();
            } catch (Exception e) {
                if (e.getMessage().contains("The PushConsumer service state not OK, maybe started once, SHUTDOWN_ALREADY")) {
                    log.warn("PushConsumer服务状态不正常，可能启动过一次，已关闭");
                    defaultMQPushConsumer.shutdown();
                    defaultMQPushConsumer.start();
                }
            }
            return waitMsg(uuid);
        } catch (Exception e) {
            log.error("消费异常", e);
            return null;
        }
    }

    private ConcurrentLinkedQueue<MsgDTO> waitMsg(String uuid) throws Exception {
        if (consumerStatus.containsKey(uuid)) {
            consumerStatus.remove(uuid);
            return msgs.get(uuid);
        }
        if (msgs.containsKey(uuid)) {
            log.info("正在消费数据,uuid:{},size:{}", uuid, msgs.get(uuid).size());
        }
        Thread.sleep(10000);
        return waitMsg(uuid);
    }


    /**
     * 锁对象
     */
    private static final Object lockObj = new Object();

    /**
     * 存放不同的日期模板格式的sdf的Map
     */
    private static Map<String, ThreadLocal<SimpleDateFormat>> sdfMap = new HashMap<String, ThreadLocal<SimpleDateFormat>>();

    /**
     * 返回一个ThreadLocal的sdf,每个线程只会new一次sdf
     *
     * @param pattern
     * @return
     */
    private static SimpleDateFormat getSdf(final String pattern) {
        ThreadLocal<SimpleDateFormat> tl = sdfMap.get(pattern);
        // 此处的双重判断和同步是为了防止sdfMap这个单例被多次put重复的sdf
        if (tl == null) {
            synchronized (lockObj) {
                tl = sdfMap.get(pattern);
                if (tl == null) {
                    // 只有Map中还没有这个pattern的sdf才会生成新的sdf并放入map
                    System.out.println("put new sdf of pattern " + pattern + " to map");

                    // 这里是关键,使用ThreadLocal<SimpleDateFormat>替代原来直接new SimpleDateFormat
                    tl = new ThreadLocal<SimpleDateFormat>() {

                        @Override
                        protected SimpleDateFormat initialValue() {
                            //System.out.println("thread: " + Thread.currentThread() + " init pattern: " + pattern);
                            return new SimpleDateFormat(pattern);
                        }
                    };
                    sdfMap.put(pattern, tl);
                }
            }
        }

        return tl.get();
    }
}

调用

ConcurrentLinkedQueue<MsgDTO> msgDTOS = consumerService.consumer(new ConsumerParamTDO(
                1632806700000L, 1632807300000L ,
                "t1||t2||t3",
                "127.0.0.1:9876",
                "topic_test"
        ));