RocketMQ源码解析-PushConsumer(3)

public boolean putMessage(final List<MessageExt> msgs) {
    boolean dispatchToConsume = false;
    try {
        this.lockTreeMap.writeLock().lockInterruptibly();
        try {
            int validMsgCnt = 0;
            for (MessageExt msg : msgs) {
                MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg);
                if (null == old) {
                    validMsgCnt++;
                    this.queueOffsetMax = msg.getQueueOffset();
                }
            }
            msgCount.addAndGet(validMsgCnt);

            if (!msgTreeMap.isEmpty() && !this.consuming) {
                dispatchToConsume = true;
                this.consuming = true;
            }

            if (!msgs.isEmpty()) {
                MessageExt messageExt = msgs.get(msgs.size() - 1);
                String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET);
                if (property != null) {
                    long accTotal = Long.parseLong(property) - messageExt.getQueueOffset();
                    if (accTotal > 0) {
                        this.msgAccCnt = accTotal;
                    }
                }
            }
        }
        finally {
            this.lockTreeMap.writeLock().unlock();
        }
    }
    catch (InterruptedException e) {
        log.error("putMessage exception", e);
    }

    return dispatchToConsume;
}

 

取回来的消息将会在proceeQueue当中存放在其中的treeMap中（整个操作为了保证线程安全，全程加锁），并且在之后统计消费的数量统计。

 

 

在存放消息之后，将是消息的消费。

调用defaultMQPushConsumerImpl下的ConsumeMessageService的submitConsumeRequest()方法来消费消息。

@Override
public void submitConsumeRequest(//
        final List<MessageExt> msgs, //
        final ProcessQueue processQueue, //
        final MessageQueue messageQueue, //
        final boolean dispatchToConsume) {
    final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();
    if (msgs.size() <= consumeBatchSize) {
        ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue);
        this.consumeExecutor.submit(consumeRequest);
    }
    else {
        for (int total = 0; total < msgs.size();) {
            List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize);
            for (int i = 0; i < consumeBatchSize; i++, total++) {
                if (total < msgs.size()) {
                    msgThis.add(msgs.get(total));
                }
                else {
                    break;
                }
            }

            ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue);
            this.consumeExecutor.submit(consumeRequest);
        }
    }
}

 

在消费消息的一开始会判断消息个数与一次最高允许消费的消息条数，如果小于，那直接回封装成ConsumeRequest消费请求丢入线程池中进行消费，而一旦大于，将会按照一次最大批量处理条数进行分次处理。

 

@Override
public void run() {
    if (this.processQueue.isDropped()) {
        log.info("the message queue not be able to consume, because it's dropped {}",
            this.messageQueue);
        return;
    }

    MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener;
    ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue);
    ConsumeConcurrentlyStatus status = null;

    ConsumeMessageContext consumeMessageContext = null;
    if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
        consumeMessageContext = new ConsumeMessageContext();
        consumeMessageContext
            .setConsumerGroup(ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer
                .getConsumerGroup());
        consumeMessageContext.setMq(messageQueue);
        consumeMessageContext.setMsgList(msgs);
        consumeMessageContext.setSuccess(false);
        ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl
            .executeHookBefore(consumeMessageContext);
    }

    long beginTimestamp = System.currentTimeMillis();

    try {
        ConsumeMessageConcurrentlyService.this.resetRetryTopic(msgs);
        status = listener.consumeMessage(Collections.unmodifiableList(msgs), context);
    }
    catch (Throwable e) {
        log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",//
            RemotingHelper.exceptionSimpleDesc(e),//
            ConsumeMessageConcurrentlyService.this.consumerGroup,//
            msgs,//
            messageQueue);
    }

    long consumeRT = System.currentTimeMillis() - beginTimestamp;

    if (null == status) {
        log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}",//
            ConsumeMessageConcurrentlyService.this.consumerGroup,//
            msgs,//
            messageQueue);
        status = ConsumeConcurrentlyStatus.RECONSUME_LATER;
    }

    if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) {
        consumeMessageContext.setStatus(status.toString());
        consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status);
        ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl
            .executeHookAfter(consumeMessageContext);
    }

    ConsumeMessageConcurrentlyService.this.getConsumerStatsManager().incConsumeRT(
        ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT);

    if (!processQueue.isDropped()) {
        ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this);
    }
    else {
        log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}",
            messageQueue, msgs);
    }
}

在这里将会在线程里调用用户配置的linstener对消息进行消费处理，在消费完毕之后，将会调用processConsumeResult()方法对消费结果进行处理。

public void processConsumeResult(//
        final ConsumeConcurrentlyStatus status, //
        final ConsumeConcurrentlyContext context, //
        final ConsumeRequest consumeRequest//
) {
    int ackIndex = context.getAckIndex();

    if (consumeRequest.getMsgs().isEmpty())
        return;

    switch (status) {
    case CONSUME_SUCCESS:
        if (ackIndex >= consumeRequest.getMsgs().size()) {
            ackIndex = consumeRequest.getMsgs().size() - 1;
        }
        int ok = ackIndex + 1;
        int failed = consumeRequest.getMsgs().size() - ok;
        this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup,
            consumeRequest.getMessageQueue().getTopic(), ok);
        this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup,
            consumeRequest.getMessageQueue().getTopic(), failed);
        break;
    case RECONSUME_LATER:
        ackIndex = -1;
        this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup,
            consumeRequest.getMessageQueue().getTopic(), consumeRequest.getMsgs().size());
        break;
    default:
        break;
    }

    switch (this.defaultMQPushConsumer.getMessageModel()) {
    case BROADCASTING:
        for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
            MessageExt msg = consumeRequest.getMsgs().get(i);
            log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString());
        }
        break;
    case CLUSTERING:
        List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size());
        for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) {
            MessageExt msg = consumeRequest.getMsgs().get(i);
            boolean result = this.sendMessageBack(msg, context);
            if (!result) {
                msg.setReconsumeTimes(msg.getReconsumeTimes() + 1);
                msgBackFailed.add(msg);
            }
        }

        if (!msgBackFailed.isEmpty()) {
            consumeRequest.getMsgs().removeAll(msgBackFailed);

            this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(),
                consumeRequest.getMessageQueue());
        }
        break;
    default:
        break;
    }

    long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs());
    if (offset >= 0) {
        this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(),
            offset, true);
    }
}

在这里，如果消息消费成功，则将在这里进行对ack也就是消费成功的消息数量进行赋值。

在广播模式下，即使消息消费失败，也只是进行日志记录，并没有别的操作。

在集群模式下，如果消息消费失败，将会先尝试将消息发送回broker，如果发送回给broker也失败了，那么将会将这条消息在一定时间后重新尝试消费，否则就从消息失败数组中移除这个消息。

最后，更新消息队列的消费进度。PushConsumer的消息消费结束。