16.5 消息的处理与消费
不多废话，直接上源码，下面这段代码是在DefaultMQPushConsumerImpl.pullMessage方法中的，用做拉取消息后回调用的。
PullCallback pullCallback = new PullCallback() {
@Override
// 拉取成功回调
public void onSuccess(PullResult pullResult) {
if (pullResult != null) {
// 主要就是对拉取的消息进行tags过滤
pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult,
subscriptionData);
switch (pullResult.getPullStatus()) {
case FOUND: // 成功的，看这块就行
// 本次请求的起始偏移
long prevRequestOffset = pullRequest.getNextOffset();
// 下一次请求的起始偏移，实际上就是本次偏移的最大值
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// pullRT ，拉取总响应时间戳
long pullRT = System.currentTimeMillis() - beginTimestamp;
// 统计相关，记录拉取次数+1，并增加拉取响应+pullRT
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullRT);
long firstMsgOffset = Long.MAX_VALUE;
if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {// 没拉到数据，那再来一次
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
} else {
// 集合中第一条数据的偏移
firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
// 统计相关，记录拉取消息总占大小，并增加到统计中
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
// 将消息数量和所占空间大小记录到缓存中，同时按所在cq队列的偏移大小排序，存储到树型结构的map中，后面会有用到
boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
// 这里开始，将消息转到消费者，开始消费，继续往里跟
DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(
pullResult.getMsgFoundList(),
processQueue,
pullRequest.getMessageQueue(),
dispatchToConsume);
// 判断有没有设置每次拉取消息后的间隔时间，默认没有，如果有的话，得间隔一段时间再去
if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
// 延迟执行下一次的拉取
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,
DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
} else {
// 没有延迟，立即执行下一次的拉取
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
}
}
// 判断非法偏移值
if (pullResult.getNextBeginOffset() < prevRequestOffset
|| firstMsgOffset < prevRequestOffset) {
log.warn(
"[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}",
pullResult.getNextBeginOffset(),
firstMsgOffset,
prevRequestOffset);
}
break;
case NO_NEW_MSG:
// 服务端没有新消息产生，更新缓存中的偏移值，继续下一次执行
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
break;
case NO_MATCHED_MSG:
// 服务端没有匹配的消息产生，更新缓存中的偏移值，继续下一次执行
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest);
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
break;
case OFFSET_ILLEGAL:
// 要拉取的消息的偏移值非法，设置新的偏移值，继续执行（本次延迟 10s 异步执行）
log.warn("the pull request offset illegal, {} {}",
pullRequest.toString(), pullResult.toString());
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
// 设置本次pq失效
pullRequest.getProcessQueue().setDropped(true);
DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() {
@Override
public void run() {
try {
// 将有效的新的偏移值更新到缓存
DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(),
pullRequest.getNextOffset(), false);
// 将新的偏移值持久化到broker存放
DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue());
// 移除失效的pq
DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue());
log.warn("fix the pull request offset, {}", pullRequest);
} catch (Throwable e) {
log.error("executeTaskLater Exception", e);
}
}
}, 10000);
break;
default:
break;
}
}
}
@Override
// 拉取异常回调
public void onException(Throwable e) {
if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("execute the pull request exception", e);
}
// 异常了，留待下一次执行
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION);
}
};
PullAPIWrapper.processPullResult
public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult,
final SubscriptionData subscriptionData) {
PullResultExt pullResultExt = (PullResultExt) pullResult;
// 更新此次请求是从哪个broker节点（brokerId）拉取的，修改的是缓存中 pullFromWhichNodeTable map的值，由brokerName、topic、queueId三者唯一确定
this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
// 表示拉取到了数据
if (PullStatus.FOUND == pullResult.getPullStatus()) {
// 将消息的二进制转换成ByteBuffer
ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());
// 解析成消息集合，有兴趣的读者可以看看怎么解析，具体解析过程就是把消息在commitlog中怎么存进去的，就怎么解析出来，参考`章节11.3.4`
List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);
List<MessageExt> msgListFilterAgain = msgList;
// 这里增加对tags的过滤，现在知道tags的用处了吧
if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) {
msgListFilterAgain = new ArrayList<MessageExt>(msgList.size());
for (MessageExt msg : msgList) { // 添加遍历
if (msg.getTags() != null) {
// 包括tags的就添加到新的消息集合 msgListFilterAgain
if (subscriptionData.getTagsSet().contains(msg.getTags())) {
msgListFilterAgain.add(msg);
}
}
}
}
// 消息过滤钩子，用户可自定义，实现接口 FilterMessageHook 就行
if (this.hasHook()) {
FilterMessageContext filterMessageContext = new FilterMessageContext();
filterMessageContext.setUnitMode(unitMode);
filterMessageContext.setMsgList(msgListFilterAgain);
this.executeHook(filterMessageContext);
}
for (MessageExt msg : msgListFilterAgain) {
// 针对事务消息的处理，忽略
String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (traFlag != null && Boolean.parseBoolean(traFlag)) {
msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
}
// 设置消息的最小偏移属性值，由于 pullResult 是入参，所以所有的消息msg的最小偏移值都是一样的
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET,
Long.toString(pullResult.getMinOffset()));
// 设置消息的最大偏移属性值，由于 pullResult 是入参，所以所有的消息msg的最大偏移值都是一样的
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET,
Long.toString(pullResult.getMaxOffset()));
}
// 把最终过滤后的消息返回到上一层调用，用户最终消费的也是这个
pullResultExt.setMsgFoundList(msgListFilterAgain);
}
// 消息的二进制已经没有用处，可以置空了
pullResultExt.setMessageBinary(null);
// 返回结果
return pullResult;
}
ConsumeMessageConcurrentlyService.submitConsumeRequest
这里只讲并发执行，顺序执行的暂时忽略
public void submitConsumeRequest(
final List<MessageExt> msgs,
final ProcessQueue processQueue,
final MessageQueue messageQueue,
final boolean dispatchToConsume) {
// 批量消息的个数，默认是1，这个用户使用时可以设置
final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();
if (msgs.size() <= consumeBatchSize) { // 拉取的消息数量小于批次限制
ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);
try {
// 提交线程池来执行
this.consumeExecutor.submit(consumeRequest);
} catch (RejectedExecutionException e) {
this.submitConsumeRequestLater(consumeRequest);
}
} else { // 拉取的消息数量大于批次限制
// 遍历集合，取 consumeBatchSize 个消息，分批次组装成新的集合，去消费
for (int total = 0; total < msgs.size(); ) {
List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize);
for (int i = 0; i < consumeBatchSize; i++, total++) {
if (total < msgs.size()) {
msgThis.add(msgs.get(total));
} else {
break;
}
}
ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);
try {
this.consumeExecutor.submit(consumeRequest);
} catch (RejectedExecutionException e) {
for (; total < msgs.size(); total++) {
msgThis.add(msgs.get(total));
}
this.submitConsumeRequestLater(consumeRequest);
}
}
}
}
ConsumeRequest.run
最后一层皮了，扒完这层，就是用户可见的了。
public void run() {
if (this.processQueue.isDropped()) {
log.info("the message queue not be able to consume, because it's dropped. group={} {}", ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue);
return;
}
// 这个监听器就是示例中，通过 registerMessageListener 方法注册进去的
MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener;
ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue);
ConsumeConcurrentlyStatus status = null;
// 将retry重推消息的topic重置为源topic
defaultMQPushConsumerImpl.resetRetryAndNamespace(msgs, defaultMQPushConsumer.getConsumerGroup());
// 钩子，用户自定义，有兴趣自行研究
ConsumeMessageContext consumeMessageContext = null;
if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
consumeMessageContext = new ConsumeMessageContext();
consumeMessageContext.setNamespace(defaultMQPushConsumer.getNamespace());
consumeMessageContext.setConsumerGroup(defaultMQPushConsumer.getConsumerGroup());
consumeMessageContext.setProps(new HashMap<String, String>());
consumeMessageContext.setMq(messageQueue);
consumeMessageContext.setMsgList(msgs);
consumeMessageContext.setSuccess(false);
ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext);
}
long beginTimestamp = System.currentTimeMillis();
boolean hasException = false;
ConsumeReturnType returnType = ConsumeReturnType.SUCCESS;
try {
if (msgs != null && !msgs.isEmpty()) {
for (MessageExt msg : msgs) {
// 给每条消息设置一个消费时间，用户可以取出来用做他用
MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis()));
}
}
// 调用监听器方法，示例中有实现，这个方法就是用户要自己实现的消费逻辑，并返回一个状态
status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
} catch (Throwable e) {
log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",
RemotingHelper.exceptionSimpleDesc(e),
ConsumeMessageConcurrentlyService.this.consumerGroup,
msgs,
messageQueue);
hasException = true; // 异常标志
}
long consumeRT = System.currentTimeMillis() - beginTimestamp;
if (null == status) { // 可能是用户不规范操作，也可能是消费过程产生了异常，导致状态为null
if (hasException) {
// 有异常，标志返回类型为异常
returnType = ConsumeReturnType.EXCEPTION;
} else {
// 无异常，标志返回类型null
returnType = ConsumeReturnType.RETURNNULL;
}
} else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) {
// 超过1分钟，标记超时
returnType = ConsumeReturnType.TIME_OUT;
} else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) {
// 重复消费，标志消费失败
returnType = ConsumeReturnType.FAILED;
} else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) {
// 消费成功
returnType = ConsumeReturnType.SUCCESS;
}
// 钩子
if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name());
}
if (null == status) {
log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}",
ConsumeMessageConcurrentlyService.this.consumerGroup,
msgs,
messageQueue);
status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
}
// 钩子
if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
consumeMessageContext.setStatus(status.toString());
consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status);
ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext);
}
// 统计相关，增加消费响应时间
ConsumeMessageConcurrentlyService.this.getConsumerStatsManager()
.incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT);
if (!processQueue.isDropped()) {
// 最后扫尾，处理消费后的结果
ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this);
} else {
log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}", messageQueue, msgs);
}
}
ConsumeMessageConcurrentlyService.processConsumeResult
public void processConsumeResult(
final ConsumeConcurrentlyStatus status,
final ConsumeConcurrentlyContext context,
final ConsumeRequest consumeRequest
) {
// 确认消费的index，默认是integer的最大值
int ackIndex = context.getAckIndex();
// 消息列表判空，默认肯定不为空了
if (consumeRequest.getMsgs().isEmpty())
return;
switch (status) {
case CONSUME_SUCCESS: // 消费成功
// 计算 ackIndex，这个值很重要的，没计算好，消息可能会作为未消费的，会导致消息重推
if (ackIndex >= consumeRequest.getMsgs().size()) {// 大于列表长度
// 设置为列表长度 - 1
ackIndex = consumeRequest.getMsgs().size() - 1;
}
// 这里+1，是因为index从0开始，表示消费成功的量
int ok = ackIndex + 1;
// 判断消费总数，与收到的总数是否一致，一致的话 failed 就是0，也即 false，就是成功
int failed = consumeRequest.getMsgs().size() - ok;
// 统计相关，计算消费成功的量
this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok);
// 统计相关，计算失败的量
this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed);
break;
case RECONSUME_LATER:
ackIndex = -1; // 表示消费要重推，设置为 -1，配合下面消息重推时使用ackIndex + 1;
this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(),
consumeRequest.getMsgs().size());
break;
default:
break;
}
switch (this.defaultMQPushConsumer.getMessageModel()) {
case BROADCASTING: // 广播模式，忽略
for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
MessageExt msg = consumeRequest.getMsgs().get(i);
log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString());
}
break;
case CLUSTERING: // 集群模式
// 封装消息重推（这个重推的意思是指消费端将消息重新发回到broker）失败集合
List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size());
// 从 ackIndex + 1 开始，这里就可以看出 ackIndex 的计算的重要性了
for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
// 取出未被消费的消息，重推到broker
MessageExt msg = consumeRequest.getMsgs().get(i);
// 将消息重新发送到broker，这个会专门抽出一个章节来讲解
boolean result = this.sendMessageBack(msg, context);
if (!result) {
// 重推失败，记录到 msgBackFailed 中
msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
msgBackFailed.add(msg);
}
}
if (!msgBackFailed.isEmpty()) {
// 从 msgs 列表中删除
consumeRequest.getMsgs().removeAll(msgBackFailed);
// 重新放到客户端消费
this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue());
}
break;
default:
break;
}
// 清空本次消费的消息，并返回剩下未消费的消息的最小偏移值
long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());
if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) {
// 更新下次要消费的偏移值
this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true);
}
}
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
