批量顺序发送
用法:
RocketMQUtil.syncSendOrderly(rocketMqTemplate, ORDER_LEDGER_HASH, messages, OrderLedgerMessageDTO::getPlatformOrderNumber);
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.producer.MessageQueueSelector;
import org.apache.rocketmq.client.producer.selector.SelectMessageQueueByHash;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.util.CollectionUtils;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* @author ly-chn
*/
@SuppressWarnings({"AlibabaConstantFieldShouldBeUpperCase", "AlibabaLowerCamelCaseVariableNaming"})
@Slf4j
public class RocketMQUtil {
/**
* 获取 DefaultMQProducerImpl#tryToFindTopicPublishInfo 方法, 解析topic信息
*/
private static final Method tryToFindTopicPublishInfo;
/**
* 消息选择器
*/
private static final MessageQueueSelector queueSelector = new SelectMessageQueueByHash();
/**
* 发送失败重试次数
*/
private static final int retryTimesWhenSendFailed = 3;
static {
try {
tryToFindTopicPublishInfo = DefaultMQProducerImpl.class
.getDeclaredMethod("tryToFindTopicPublishInfo", String.class);
tryToFindTopicPublishInfo.setAccessible(true);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
/**
* 批量顺序发送消息
*
* @see RocketMQUtil#buildMap(RocketMQTemplate, String, List, Function)
*/
public static <T> Map<String, List<Message<T>>> syncSendOrderly(RocketMQTemplate mqTemplate, String topic, List<T> messageList, Function<T, String> hashKeyGetter) {
Map<String, List<Message<T>>> map = buildMap(mqTemplate, topic, messageList, hashKeyGetter);
log.info("发送{}消息 count={}, hashkey={},messageList={}", topic, messageList.size(), hashKeyGetter, JSON.toJSONString(messageList));
for (String key : map.keySet()) {
try {
List<Message<T>> list = map.get(key);
boolean success= false;
for (int i = 0; i < retryTimesWhenSendFailed; i++) {
try {
mqTemplate.syncSendOrderly(topic, list, key, 600000);
success = true;
break;
} catch (Exception e) {
log.error("发送消息失败, retry: {},topic={}, key={}, value={}", i, topic, key, JSON.toJSONString(list.stream().map(Message::getPayload)), e);
}
}
if (!success) {
map.remove(key);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return map;
}
/**
* 解析批量顺序发送消息
*
* @param mqTemplate RocketMQTemplate对象
* @param topic 消息topic
* @param messageList 要发送到消息列表
* @param hashKeyGetter 获取hashKey的函数
* @param <T> 消息类型
* @return 待发送的消息列表
*/
public static <T> Map<String, List<Message<T>>> buildMap(RocketMQTemplate mqTemplate, String topic, List<T> messageList, Function<T, String> hashKeyGetter) {
// noinspection deprecation
DefaultMQProducerImpl defaultMQProducerImpl = mqTemplate.getProducer().getDefaultMQProducerImpl();
MQClientInstance mqClientInstance = defaultMQProducerImpl.getmQClientFactory();
// 读取topic信息
TopicPublishInfo topicPublishInfo;
try {
topicPublishInfo = (TopicPublishInfo) tryToFindTopicPublishInfo.invoke(defaultMQProducerImpl, topic);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e);
}
// 消息队列列表
List<MessageQueue> messageQueueList =
mqClientInstance.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList());
// 发送消息
Map<MessageQueue, List<T>> map = messageList.stream().collect(Collectors.groupingBy(it -> queueSelector.select(messageQueueList, null, hashKeyGetter.apply(it))));
Map<String, List<Message<T>>> result = new HashMap<>();
// 根据不同消息队列发送消息
map.forEach((messageQueue, list) -> {
if (CollectionUtils.isEmpty(list)) {
return;
}
List<Message<T>> msgList = list.stream().map(it -> MessageBuilder.withPayload(it).build()).collect(Collectors.toList());
result.put(hashKeyGetter.apply(list.get(0)), msgList);
log.info("buildMap queueId={}, count={}", messageQueue.getQueueId(), list.size());
});
return result;
}
}
批量顺序消费
如果为并发消费, 配置MessageListenerOrderly为MessageListenerConcurrently即可
异常捕获改为
# 延迟模式, -1: 不再消费, 0: broker控制, 大于0: 自己控制, 见常见问题 context.setDelayLevelWhenNextConsume(1); # 稍后消费 return ConsumeConcurrentlyStatus.RECONSUME_LATER;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.spring.annotation.ConsumeMode;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.spring.core.RocketMQPushConsumerLifecycleListener;
import org.springframework.stereotype.Component;
/**
* 通过注解的方式实现消费者注入, 然后通过覆盖MessageListener来实现批量消费
*
* @author ly-chn
*/
@Component
@Slf4j
@RocketMQMessageListener(topic = "ly_test_1", consumerGroup = "ly_test_consumer_1", consumeMode = ConsumeMode.ORDERLY)
@RequiredArgsConstructor
public class BatchConsumer implements RocketMQListener<String>, RocketMQPushConsumerLifecycleListener {
@Override
public void onMessage(String message) {
// component仅用于注册, 此方法后续会被覆盖, 无需关注
}
@Override
public void prepareStart(DefaultMQPushConsumer consumer) {
// 每次批量拉取消息的大小
consumer.setPullBatchSize(100);
// 每次消费消息的大小
consumer.setConsumeMessageBatchMaxSize(100);
// 具体要注册的实现类, 需要与注解中consumeMode保持一致, 否则将会走默认的MessageListener
consumer.registerMessageListener((MessageListenerOrderly) (msgs, context) -> {
try {
log.info("订单台账本次拉取数据量:count={}", messages.size());
// 具体处理逻辑
} catch (Exception e) {
log.warn("consume message failed.messages:{}", messages, e);
// 消费失败, 暂停1000毫秒后继续消费
context.setSuspendCurrentQueueTimeMillis(1000);
return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
}
// 消费成功
return ConsumeOrderlyStatus.SUCCESS;
});
}
}
问题
-
每次拉取消息大小设置32以上不生效
修改broker配置, 需要重启broker
# 每次最多拉取的条数 maxTransferCountOnMessageInMemory=5000 # 每次最多拉取的空间大小(b) maxTransferBytesOnMessageInMemory = 5000 * 1024
-
消费失败重试时长(并发消费需要, 顺序消费不需要)
默认broker控制值为: 消息重试
如果时自己配置: 延时消息约束
可以修改broker配置
messageDelayLevel=30s 10m 30m 1h 2h 10h 24h 48h 7d
-
重复消费批次问题
并发消费如果消费失败的可能性较大, 需要自己处理消费批次问题