RocketMQ 批量顺序发送&消费

批量顺序发送

用法:

RocketMQUtil.syncSendOrderly(rocketMqTemplate, ORDER_LEDGER_HASH, messages, OrderLedgerMessageDTO::getPlatformOrderNumber);

import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.selector.SelectMessageQueueByHash;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.util.CollectionUtils;

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
 * @author ly-chn
 */
@SuppressWarnings({"AlibabaConstantFieldShouldBeUpperCase", "AlibabaLowerCamelCaseVariableNaming"})
@Slf4j
public class RocketMQUtil {

    /**
     * 获取 DefaultMQProducerImpl#tryToFindTopicPublishInfo 方法, 解析topic信息
     */
    private static final Method tryToFindTopicPublishInfo;
    /**
     * 消息选择器
     */
    private static final MessageQueueSelector queueSelector = new SelectMessageQueueByHash();
    /**
     * 发送失败重试次数
     */
    private static final int retryTimesWhenSendFailed = 3;

    static {
        try {
            tryToFindTopicPublishInfo = DefaultMQProducerImpl.class
                    .getDeclaredMethod("tryToFindTopicPublishInfo", String.class);
            tryToFindTopicPublishInfo.setAccessible(true);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * 批量顺序发送消息
     *
     * @see RocketMQUtil#buildMap(RocketMQTemplate, String, List, Function)
     */
    public static <T> Map<String, List<Message<T>>> syncSendOrderly(RocketMQTemplate mqTemplate, String topic, List<T> messageList, Function<T, String> hashKeyGetter) {
        Map<String, List<Message<T>>> map = buildMap(mqTemplate, topic, messageList, hashKeyGetter);
        log.info("发送{}消息 count={}, hashkey={}，messageList={}", topic, messageList.size(), hashKeyGetter, JSON.toJSONString(messageList));
        for (String key : map.keySet()) {
            try {
                List<Message<T>> list = map.get(key);
                boolean success= false;
                for (int i = 0; i < retryTimesWhenSendFailed; i++) {
                    try {
                        mqTemplate.syncSendOrderly(topic, list, key, 600000);
                        success = true;
                        break;
                    } catch (Exception e) {
                        log.error("发送消息失败, retry: {}，topic={}, key={}, value={}", i, topic, key, JSON.toJSONString(list.stream().map(Message::getPayload)), e);
                    }
                }
                if (!success) {
                    map.remove(key);
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        return map;
    }

    /**
     * 解析批量顺序发送消息
     *
     * @param mqTemplate    RocketMQTemplate对象
     * @param topic         消息topic
     * @param messageList   要发送到消息列表
     * @param hashKeyGetter 获取hashKey的函数
     * @param <T>           消息类型
     * @return 待发送的消息列表
     */
    public static <T> Map<String, List<Message<T>>> buildMap(RocketMQTemplate mqTemplate, String topic, List<T> messageList, Function<T, String> hashKeyGetter) {
        // noinspection deprecation
        DefaultMQProducerImpl defaultMQProducerImpl = mqTemplate.getProducer().getDefaultMQProducerImpl();
        MQClientInstance mqClientInstance = defaultMQProducerImpl.getmQClientFactory();
        // 读取topic信息
        TopicPublishInfo topicPublishInfo;
        try {
            topicPublishInfo = (TopicPublishInfo) tryToFindTopicPublishInfo.invoke(defaultMQProducerImpl, topic);
        } catch (IllegalAccessException | InvocationTargetException e) {
            throw new RuntimeException(e);
        }
        // 消息队列列表
        List<MessageQueue> messageQueueList =
                mqClientInstance.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());
        // 发送消息
        Map<MessageQueue, List<T>> map = messageList.stream().collect(Collectors.groupingBy(it -> queueSelector.select(messageQueueList, null, hashKeyGetter.apply(it))));
        Map<String, List<Message<T>>> result = new HashMap<>();
        // 根据不同消息队列发送消息
        map.forEach((messageQueue, list) -> {
            if (CollectionUtils.isEmpty(list)) {
                return;
            }
            List<Message<T>> msgList = list.stream().map(it -> MessageBuilder.withPayload(it).build()).collect(Collectors.toList());
            result.put(hashKeyGetter.apply(list.get(0)), msgList);
            log.info("buildMap queueId={}, count={}", messageQueue.getQueueId(), list.size());
        });
        return result;
    }

}

批量顺序消费

如果为并发消费, 配置MessageListenerOrderly为MessageListenerConcurrently即可

异常捕获改为

# 延迟模式, -1: 不再消费, 0: broker控制, 大于0: 自己控制, 见常见问题
context.setDelayLevelWhenNextConsume(1);
# 稍后消费
return ConsumeConcurrentlyStatus.RECONSUME_LATER;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.spring.annotation.ConsumeMode;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.spring.core.RocketMQPushConsumerLifecycleListener;
import org.springframework.stereotype.Component;

/**
 * 通过注解的方式实现消费者注入, 然后通过覆盖MessageListener来实现批量消费
 *
 * @author ly-chn
 */
@Component
@Slf4j
@RocketMQMessageListener(topic = "ly_test_1", consumerGroup = "ly_test_consumer_1", consumeMode = ConsumeMode.ORDERLY)
@RequiredArgsConstructor
public class BatchConsumer implements RocketMQListener<String>, RocketMQPushConsumerLifecycleListener {

    @Override
    public void onMessage(String message) {
        // component仅用于注册, 此方法后续会被覆盖, 无需关注
    }


    @Override
    public void prepareStart(DefaultMQPushConsumer consumer) {
        // 每次批量拉取消息的大小
        consumer.setPullBatchSize(100);
        // 每次消费消息的大小
        consumer.setConsumeMessageBatchMaxSize(100);
        // 具体要注册的实现类, 需要与注解中consumeMode保持一致, 否则将会走默认的MessageListener
        consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {
                        try {
                log.info("订单台账本次拉取数据量：count={}", messages.size());
                // 具体处理逻辑
            } catch (Exception e) {
                log.warn("consume message failed.messages:{}", messages, e);
                // 消费失败, 暂停1000毫秒后继续消费
                context.setSuspendCurrentQueueTimeMillis(1000);
                return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
            }
            // 消费成功
            return ConsumeOrderlyStatus.SUCCESS;
        });
    }
}

问题

1. 每次拉取消息大小设置32以上不生效

   修改broker配置, 需要重启broker

   # 每次最多拉取的条数
   maxTransferCountOnMessageInMemory=5000
   # 每次最多拉取的空间大小(b)
   maxTransferBytesOnMessageInMemory = 5000 * 1024
   

2. 消费失败重试时长(并发消费需要, 顺序消费不需要)

   默认broker控制值为: 消息重试

   如果时自己配置: 延时消息约束

   可以修改broker配置

   messageDelayLevel=30s 10m 30m 1h 2h 10h 24h 48h 7d
   

3. 重复消费批次问题

并发消费如果消费失败的可能性较大, 需要自己处理消费批次问题