rocketmq核心源码分析第十八篇一消息消费五部曲一服务端消息拉取及MessageArrivingListener-PullRequestHoldService消息挂起

PullMessageProcessor处理消息拉取请求

核心流程
基于过滤信息构建tag,sql92等消息过滤器
基于消息过滤器进行消息拉取
处理长轮询机制
消息进度上报存储

|-PullMessageProcessor一构建过滤器原理图

|-PullMessageProcessor.processRequest一构建消息过滤器

• 消息过滤分为tag过滤,sql92过滤以及filter过滤
• filter会新建filterserver进程进行代理,提高cpu资源利用率,但后期应该会废除[本篇章不在介绍classfilter类过滤模式]

• 存在子订阅,构建sql92或tag模式的SubscriptionData以及ConsumerFilterData
• 不存在子订阅,构建classfilter模式的SubscriptionData以及ConsumerFilterData
• 根据SubscriptionData以及ConsumerFilterData构建消息过滤器MessageFilter

private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
            throws RemotingCommandException {
    RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
    final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
    final PullMessageRequestHeader requestHeader =
            (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class);
    response.setOpaque(request.getOpaque());
    log.debug("receive PullMessage request command, {}", request);
    // 只读模式则直接返回
    if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
        response.setCode(ResponseCode.NO_PERMISSION);
        response.setRemark(String.format("the broker[%s] pulling message is forbidden", this.brokerController.getBrokerConfig().getBrokerIP1()));
        return response;
    }
    // 获取消费组订阅相关信息
    SubscriptionGroupConfig subscriptionGroupConfig =
            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
    if (null == subscriptionGroupConfig) {
        response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
        response.setRemark(String.format("subscription group [%s] does not exist, %s", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
        return response;
    }
    // 判断消费权限
    if (!subscriptionGroupConfig.isConsumeEnable()) {
        response.setCode(ResponseCode.NO_PERMISSION);
        response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
        return response;
    }
    // 根据请求头SysFlag解析出是否允许suspend
    final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag());
    // 是否持久化消费进度
    final boolean hasCommitOffsetFlag = PullSysFlag.hasCommitOffsetFlag(requestHeader.getSysFlag());
    final boolean hasSubscriptionFlag = PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag());
    // 15s
    final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;

    TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
    if (null == topicConfig) {
        log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
        response.setCode(ResponseCode.TOPIC_NOT_EXIST);
        response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
        return response;
    }

    if (!PermName.isReadable(topicConfig.getPerm())) {
        response.setCode(ResponseCode.NO_PERMISSION);
        response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden");
        return response;
    }

    if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
        String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
        log.warn(errorInfo);
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(errorInfo);
        return response;
    }

    // 过滤表达式类模式相关信息[tag,sql92]
    SubscriptionData subscriptionData = null;
    // 过滤类或者sql92过滤相关信息
    ConsumerFilterData consumerFilterData = null;
    // 如果存在子订阅 比如tag  sql92
    if (hasSubscriptionFlag) {
        try {
            // 根据消费组，订阅模式，消息过滤模式构建订阅模式 SubscriptionData
            subscriptionData = FilterAPI.build(
                    requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType()
            );
            // 表示sql92模式
            if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
                // 构建sql92 过滤信息
                consumerFilterData = ConsumerFilterManager.build(
                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(),
                        requestHeader.getExpressionType(), requestHeader.getSubVersion()
                );
                assert consumerFilterData != null;
            }
        } catch (Exception e) {
            log.warn("Parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(),
                    requestHeader.getConsumerGroup());
            response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
            response.setRemark("parse the consumer's subscription failed");
            return response;
        }
    } else {
        // 无子订阅模式
        /*
        走的是ClassFilter过滤模式，此时不是构建SubscriptionData,
        而是直接从brokerController.getConsumerFilterManager() 中根据 topic、consumerGroup或取，
        如果取不到直接提示错误，为什么会这样呢？原来在调用subscribe(String topic, String fullClassName, String filterClassSource) 方法时，会创建相关的订阅信息
         */
        ConsumerGroupInfo consumerGroupInfo =
                this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
        if (null == consumerGroupInfo) {
            log.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
            response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
            response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
            return response;
        }

        if (!subscriptionGroupConfig.isConsumeBroadcastEnable()
                && consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) {
            response.setCode(ResponseCode.NO_PERMISSION);
            response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] can not consume by broadcast way");
            return response;
        }

        subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic());
        if (null == subscriptionData) {
            log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic());
            response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
            response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
            return response;
        }

        if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) {
            log.warn("The broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(),
                    subscriptionData.getSubString());
            response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST);
            response.setRemark("the consumer's subscription not latest");
            return response;
        }
        // 构建filterclass信息
        if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
            consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(),
                    requestHeader.getConsumerGroup());
            if (consumerFilterData == null) {
                response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST);
                response.setRemark("The broker's consumer filter data is not exist!Your expression may be wrong!");
                return response;
            }
            if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {
                log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
                        requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
                response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);
                response.setRemark("the consumer's consumer filter data not latest");
                return response;
            }
        }
    }
    // 消息过滤模式为 SQL92 ，则必须在broker端开启 enablePropertyFilter=true。
    if (!ExpressionType.isTagType(subscriptionData.getExpressionType())
            && !this.brokerController.getBrokerConfig().isEnablePropertyFilter()) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark("The broker does not support consumer to filter message by " + subscriptionData.getExpressionType());
        return response;
    }
    // 构建过滤器
    // 根据是否可以重试broker、filterSupportRetry，创建 ExpressionForRetryMessageFilter、ExpressionMessageFilter 消息过滤器
    MessageFilter messageFilter;
    if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) {
        messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData,
                this.brokerController.getConsumerFilterManager());
    } else {
        messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,
                this.brokerController.getConsumerFilterManager());
    }
    ...... 删除消息拉取以及suspend机制长短轮询处理
}

||- PullMessageProcessor.processRequest一消息拉取原理图

||- PullMessageProcessor.processRequest一消息拉取

• 进行消息拉取
• 处理响应头
• 执行hook

private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
        throws RemotingCommandException {
    ...... 删除过滤器构建代码
    step-2: 核心处理通过defaultmessageStore 拉取消息
    final GetMessageResult getMessageResult =
        this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
                requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter/* 根据consumerFilterData 构建messageFilter 实现消息过滤*/);
    if (getMessageResult != null) {
    // step-2.1: 参数设置
    response.setRemark(getMessageResult.getStatus().name());
    responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());
    responseHeader.setMinOffset(getMessageResult.getMinOffset());
    responseHeader.setMaxOffset(getMessageResult.getMaxOffset());

    if (getMessageResult.isSuggestPullingFromSlave()) {
        responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
    } else {
        responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
    }

    switch (this.brokerController.getMessageStoreConfig().getBrokerRole()) {
        case ASYNC_MASTER:
        case SYNC_MASTER:
            break;
        case SLAVE:
            // 从机不允许则立刻重试master
            if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
                response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
                responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
            }
            break;
    }

    if (this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
        // consume too slow ,redirect to another machine
        if (getMessageResult.isSuggestPullingFromSlave()) {
            responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
        }
        // consume ok
        else {
            responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
        }
    } else {
        // 如果从节点不可读 则建议从主节点拉取
        responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
    }

    switch (getMessageResult.getStatus()) {
        case FOUND:
            response.setCode(ResponseCode.SUCCESS);
            break;
        case MESSAGE_WAS_REMOVING:
            response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
            break;
        case NO_MATCHED_LOGIC_QUEUE:
        case NO_MESSAGE_IN_QUEUE:
            if (0 != requestHeader.getQueueOffset()) {
                response.setCode(ResponseCode.PULL_OFFSET_MOVED);

                // XXX: warn and notify me
                log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",
                        requestHeader.getQueueOffset(),
                        getMessageResult.getNextBeginOffset(),
                        requestHeader.getTopic(),
                        requestHeader.getQueueId(),
                        requestHeader.getConsumerGroup()
                );
            } else {
                response.setCode(ResponseCode.PULL_NOT_FOUND);
            }
            break;
        case NO_MATCHED_MESSAGE:
            response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
            break;
        case OFFSET_FOUND_NULL:
            response.setCode(ResponseCode.PULL_NOT_FOUND);
            break;
        case OFFSET_OVERFLOW_BADLY:
            response.setCode(ResponseCode.PULL_OFFSET_MOVED);
            // XXX: warn and notify me
            log.info("the request offset: {} over flow badly, broker max offset: {}, consumer: {}",
                    requestHeader.getQueueOffset(), getMessageResult.getMaxOffset(), channel.remoteAddress());
            break;
        case OFFSET_OVERFLOW_ONE:
            response.setCode(ResponseCode.PULL_NOT_FOUND);
            break;
        case OFFSET_TOO_SMALL:
            response.setCode(ResponseCode.PULL_OFFSET_MOVED);
            log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}",
                    requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(),
                    getMessageResult.getMinOffset(), channel.remoteAddress());
            break;
        default:
            assert false;
            break;
    }
    step-2.2: 消息hook处理
    if (this.hasConsumeMessageHook()) {
        ConsumeMessageContext context = new ConsumeMessageContext();
        context.setConsumerGroup(requestHeader.getConsumerGroup());
        context.setTopic(requestHeader.getTopic());
        context.setQueueId(requestHeader.getQueueId());

        String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);

        switch (response.getCode()) {
            case ResponseCode.SUCCESS:
                int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();
                int incValue = getMessageResult.getMsgCount4Commercial() * commercialBaseCount;

                context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_SUCCESS);
                context.setCommercialRcvTimes(incValue);
                context.setCommercialRcvSize(getMessageResult.getBufferTotalSize());
                context.setCommercialOwner(owner);

                break;
            没有找到消息
            case ResponseCode.PULL_NOT_FOUND:
                if (!brokerAllowSuspend) {

                    context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);
                    context.setCommercialRcvTimes(1);
                    context.setCommercialOwner(owner);

                }
                break;
            case ResponseCode.PULL_RETRY_IMMEDIATELY:
            case ResponseCode.PULL_OFFSET_MOVED:
                context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);
                context.setCommercialRcvTimes(1);
                context.setCommercialOwner(owner);
                break;
            default:
                assert false;
                break;
        }

        this.executeConsumeMessageHookBefore(context);
    }
    ...... 删除suspend挂起代码
}

||- DefaultMessageStore.getMessage拉取批量消息

• 获取当前的ConsumeQueue
• offset位于minOffset与maxOffset之间则进行拉取
• 一次预计拉取800条
• 获取consumequeue的offset size tag
• 判断本批次消息是否结束
• broker端根据hash过滤
• 通过commitlog获取消息
• 根据commitlog进行sql92过滤
• 添加一条消息到拉取结果

public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset,
                                       final int maxMsgNums,
                                       final MessageFilter messageFilter) {
    if (this.shutdown) {
        log.warn("message store has shutdown, so getMessage is forbidden");
        return null;
    }

    if (!this.runningFlags.isReadable()) {
        log.warn("message store is not readable, so getMessage is forbidden " + this.runningFlags.getFlagBits());
        return null;
    }

    long beginTime = this.getSystemClock().now();

    GetMessageStatus status = GetMessageStatus.NO_MESSAGE_IN_QUEUE;
    long nextBeginOffset = offset;
    long minOffset = 0;
    long maxOffset = 0;

    GetMessageResult getResult = new GetMessageResult();

    final long maxOffsetPy = this.commitLog.getMaxOffset();
    获取当前的ConsumeQueue
    ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId);
    if (consumeQueue != null) {
        minOffset = consumeQueue.getMinOffsetInQueue();
        maxOffset = consumeQueue.getMaxOffsetInQueue();

        if (maxOffset == 0) { 表示队列无消息
            status = GetMessageStatus.NO_MESSAGE_IN_QUEUE;
            计算下一次拉取拉取的开始偏移量： nextBeginOffset = nextOffsetCorrection(offset, 0)
            nextBeginOffset = nextOffsetCorrection(offset, 0);
        } else if (offset < minOffset) {
            status = GetMessageStatus.OFFSET_TOO_SMALL;
            nextBeginOffset = nextOffsetCorrection(offset, minOffset);
        } else if (offset == maxOffset) {
            status = GetMessageStatus.OFFSET_OVERFLOW_ONE;
            nextBeginOffset = nextOffsetCorrection(offset, offset);
        } else if (offset > maxOffset) {
            status = GetMessageStatus.OFFSET_OVERFLOW_BADLY;
            if (0 == minOffset) {
                nextBeginOffset = nextOffsetCorrection(offset, minOffset);
            } else {
                nextBeginOffset = nextOffsetCorrection(offset, maxOffset);
            }
        } else {
            ConsumeQueue有一个minOffset和maxOffset以及当前的消费位点offset
            offset 一般位于minOffset与maxOffset之间,如果不正常则进行如上处理
            offset位于minOffset与maxOffset之间
            SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(offset);
            if (bufferConsumeQueue != null) {
                try {
                    status = GetMessageStatus.NO_MATCHED_MESSAGE;

                    long nextPhyFileStartOffset = Long.MIN_VALUE;
                    long maxPhyOffsetPulling = 0;

                    int i = 0;
                    // 16000/(8+4+8) = 800条
                    final int maxFilterMessageCount = Math.max(16000, maxMsgNums * ConsumeQueue.CQ_STORE_UNIT_SIZE);
                    final boolean diskFallRecorded = this.messageStoreConfig.isDiskFallRecorded();
                    ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
                    一次预计拉取800
                    for (; i < bufferConsumeQueue.getSize() && i < maxFilterMessageCount; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
                        获取消息在commitlog的起始偏移量
                        long offsetPy = bufferConsumeQueue.getByteBuffer().getLong();
                        获取消息在commitlog的size
                        int sizePy = bufferConsumeQueue.getByteBuffer().getInt();
                        获取消息的tag对应hashcode
                        long tagsCode = bufferConsumeQueue.getByteBuffer().getLong();

                        maxPhyOffsetPulling = offsetPy;

                        if (nextPhyFileStartOffset != Long.MIN_VALUE) {
                            if (offsetPy < nextPhyFileStartOffset)
                                continue;
                        }
                        判断拉取的消息是否在磁盘上
                        boolean isInDisk = checkInDiskByCommitOffset(offsetPy, maxOffsetPy);
                        if (this.isTheBatchFull(sizePy, maxMsgNums, getResult.getBufferTotalSize(), getResult.getMessageCount(),
                                isInDisk)) {
                            break;
                        }

                        boolean extRet = false, isTagsCodeLegal = true;
                        if (consumeQueue.isExtAddr(tagsCode)) {
                            extRet = consumeQueue.getExt(tagsCode, cqExtUnit);
                            if (extRet) {
                                tagsCode = cqExtUnit.getTagsCode();
                            } else {
                                // can't find ext content.Client will filter messages by tag also.
                                log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}, topic={}, group={}",
                                        tagsCode, offsetPy, sizePy, topic, group);
                                isTagsCodeLegal = false;
                            }
                        }
                        broker端根据hash过滤 expressionType : tag过滤类型，分为 TAG 、SQL92。 这里只能处理TAG模式
                        if (messageFilter != null
                                && !messageFilter.isMatchedByConsumeQueue(isTagsCodeLegal ? tagsCode : null, extRet ? cqExtUnit : null)) {
                            if (getResult.getBufferTotalSize() == 0) {
                                status = GetMessageStatus.NO_MATCHED_MESSAGE;
                            }
                            continue;
                        }
                        通过commitlog获取消息
                        SelectMappedBufferResult selectResult = this.commitLog.getMessage(offsetPy, sizePy);
                        if (null == selectResult) {
                            if (getResult.getBufferTotalSize() == 0) {
                                status = GetMessageStatus.MESSAGE_WAS_REMOVING;
                            }

                            nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy);
                            continue;
                        }
                        在根据commitlog重新过滤
                        broker端根据hash过滤 expressionType : tag过滤类型，分为 TAG 、SQL92。 这里只能处理SQL92模式
                        sql92 需要根据消息内容进行过滤 所以这里需要查询commitlog的具体内容
                        if (messageFilter != null

                                && /* ExpressionMessageFilter */ !messageFilter.isMatchedByCommitLog(selectResult.getByteBuffer().slice(), null)) {
                            if (getResult.getBufferTotalSize() == 0) {
                                status = GetMessageStatus.NO_MATCHED_MESSAGE;
                            }
                            // release...
                            selectResult.release();
                            continue;
                        }

                        this.storeStatsService.getGetMessageTransferedMsgCount().incrementAndGet();
                        添加一条消息
                        getResult.addMessage(selectResult);
                        status = GetMessageStatus.FOUND;
                        nextPhyFileStartOffset = Long.MIN_VALUE;
                    }
                    统计处理
                    if (diskFallRecorded) {
                        long fallBehind = maxOffsetPy - maxPhyOffsetPulling;
                        brokerStatsManager.recordDiskFallBehindSize(group, topic, queueId, fallBehind);
                    }
                    处理新的消费位点
                    nextBeginOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
                    diff 从maxPhyOffsetPulling到maxOffsetPy还没有拉取
                    long diff = maxOffsetPy - maxPhyOffsetPulling;
                    如果diff位于磁盘,建议从slave读取
                    long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE
                            * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0));
                    getResult.setSuggestPullingFromSlave(diff > memory);
                } finally {

                    bufferConsumeQueue.release();
                }
            } else {
                status = GetMessageStatus.OFFSET_FOUND_NULL;
                nextBeginOffset = nextOffsetCorrection(offset, consumeQueue.rollNextFile(offset));
                log.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: "
                        + maxOffset + ", but access logic queue failed.");
            }
        }
    } else {
        status = GetMessageStatus.NO_MATCHED_LOGIC_QUEUE;
        nextBeginOffset = nextOffsetCorrection(offset, 0);
    }

    if (GetMessageStatus.FOUND == status) {
        this.storeStatsService.getGetMessageTimesTotalFound().incrementAndGet();
    } else {
        this.storeStatsService.getGetMessageTimesTotalMiss().incrementAndGet();
    }
    long elapsedTime = this.getSystemClock().now() - beginTime;
    this.storeStatsService.setGetMessageEntireTimeMax(elapsedTime);

    getResult.setStatus(status);
    getResult.setNextBeginOffset(nextBeginOffset);
    getResult.setMaxOffset(maxOffset);
    getResult.setMinOffset(minOffset);
    return getResult;
}

|||- PullMessageProcessor.processRequest一消息挂起与长短轮询机制原理图

|||- PullMessageProcessor.processRequest一消息挂起与长短轮询机制

• 拉取消息成功则响应客户端
• 无新消息进行挂起处理
• 根据配置判断是采用长轮询挂起还是短轮询挂起

private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
            throws RemotingCommandException {
    ...... 删除其他代码
    step-2.3 核心 返回或者挂起进行长轮询或者短轮询
    switch (response.getCode()) {
        如果拉取消息正常,则响应客户端
        case ResponseCode.SUCCESS:
            this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
                    getMessageResult.getMessageCount());

            this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
                    getMessageResult.getBufferTotalSize());

            this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
            if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
                final long beginTimeMills = this.brokerController.getMessageStore().now();
                final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
                this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
                        requestHeader.getTopic(), requestHeader.getQueueId(),
                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
                response.setBody(r);
            } else {
                try {
                    FileRegion fileRegion =
                            new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
                    channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
                        @Override
                        public void operationComplete(ChannelFuture future) throws Exception {
                            getMessageResult.release();
                            if (!future.isSuccess()) {
                                log.error("transfer many message by pagecache failed, {}", channel.remoteAddress(), future.cause());
                            }
                        }
                    });
                } catch (Throwable e) {
                    log.error("transfer many message by pagecache exception", e);
                    getMessageResult.release();
                }

                response = null;
            }
            break;
        case ResponseCode.PULL_NOT_FOUND:
            没有找到消息则长轮询机制处理  

            brokerAllowSuspend第一次为true
            if (brokerAllowSuspend && hasSuspendFlag) {
                pollingTimeMills即挂起的时间也就是下次处理时间  suspendTimeoutMillisLong为15s
                long pollingTimeMills = suspendTimeoutMillisLong;

                长轮询没打开
                if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {
                    则改为短轮询挂起机制 pollingTimeMills为1s
                    pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
                }

                String topic = requestHeader.getTopic();
                long offset = requestHeader.getQueueOffset();
                int queueId = requestHeader.getQueueId();
                找不到在创建pull请求放入PullRequestHoldService
                PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
                        this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
                 延迟处理
                this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
                response = null;
                break;
            }
        case ResponseCode.PULL_RETRY_IMMEDIATELY:
            break;
        case ResponseCode.PULL_OFFSET_MOVED:
            if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE
                    || this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) {
                MessageQueue mq = new MessageQueue();
                mq.setTopic(requestHeader.getTopic());
                mq.setQueueId(requestHeader.getQueueId());
                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());

                OffsetMovedEvent event = new OffsetMovedEvent();
                event.setConsumerGroup(requestHeader.getConsumerGroup());
                event.setMessageQueue(mq);
                event.setOffsetRequest(requestHeader.getQueueOffset());
                event.setOffsetNew(getMessageResult.getNextBeginOffset());
                this.generateOffsetMovedEvent(event);
                log.warn(
                        "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}",
                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(),
                        responseHeader.getSuggestWhichBrokerId());
            } else {
                responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
                response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
                log.warn("PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}",
                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(),
                        responseHeader.getSuggestWhichBrokerId());
            }

            break;
        default:
            assert false;
    }


    boolean storeOffsetEnable = brokerAllowSuspend; // broker要允许suspend 长轮询机制 新请求允许 已经suspend的请求不可以
    storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag;
    storeOffsetEnable = storeOffsetEnable
            && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;

    除去消费者定时5s上报消费进度  pullmsg也会上报消费进度
    if (storeOffsetEnable) {
        this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel),
                requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
    }
    return response;
}   

总结

• 根据消息过滤器进行消息拉取
• 当拉取消息成功或一些异常场景则之前返回客户端
• 当允许pullRequest挂起,则拉取消息无新的未消费消息,挂起pullRequest
• 最后会存储pullRequest中上报的客户端消费进度

扩展点: 服务端为什么根据taghashcode过滤而不直接根据tag过滤

• taghashcode可以加tag从变长通过hash算法转成定长结构,从而使得consumequeue的每一个元素都定长化,形成索引文件
• 缺点,根据tag hash 过滤,还需要客户端支持tag过滤,才能形成逻辑闭环,同时存在非必要消息浪费网络带宽[hash冲突]

扩展点:源码分析一isTheBatchFull

• 预计处理的800条消息被中断规则
• 拉取32条不在拉取
• 在磁盘上判断条数或者字节数超过阈值不在拉取
• 在内存上判断条数或者字节数超过阈值不在拉取

 private boolean isTheBatchFull(int sizePy, int maxMsgNums, int bufferTotal, int messageTotal, boolean isInDisk) {

        if (0 == bufferTotal || 0 == messageTotal) {
            return false;
        }
        拉满32条
        if (maxMsgNums <= messageTotal) {
            return true;
        }
        在磁盘上
        if (isInDisk) {
            [已经拉取+待拉取消息长度]超64kb 不再拉取
            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInDisk()) {
                return true;
            }
            拉取8条则不再拉取
            if (messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInDisk() - 1) {
                return true;
            }
        } else {
            不在磁盘上
            [已经拉取+待拉取消息长度]超256kb 不再拉取
            if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInMemory()) {
                return true;
            }
            拉超32条则不在拉取
            if (messageTotal > this.messageStoreConfig.getMaxTransferCountOnMessageInMemory() - 1) {
                return true;
            }
        }

        return false;
    }

扩展点:源码分析一messageFilter机制

• filterclass机制不在介绍,将来应该会废弃
• tag过滤针对consumequeue文件
• sql92过滤针对commitlog文件

public class ExpressionMessageFilter implements MessageFilter {
    进行tag hashcode过滤
    @Override
    public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
        ...... 删除其他代码
        为tag 判断是否满足
        if (tagsCode == null) {
            return true;
        }
        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }
        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
        return true;
        ...... 删除其他代码
    }
    
    进行sql92过滤
    @Override
    public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
        ...... 删除其他代码
        Object ret = null;
        try {
            MessageEvaluationContext context = new MessageEvaluationContext(tempProperties);
            然后对表达式进行匹配，上下文环境为消息体中的属性【sql92】，如果匹配，则返回true,否则返回false。
            ret = realFilterData.getCompiledExpression().evaluate(context);
        } catch (Throwable e) {
            log.error("Message Filter error, " + realFilterData + ", " + tempProperties, e);
        }
        return (Boolean) ret;
        ...... 删除其他代码
    }

}

扩展点:源码分析一PullRequestHoldService及MessageArrivingListener

• 无新消息的pullrequest会加入PullRequestHoldService
• 短轮询,PullRequestHoldService会每隔1秒重新执行挂起的pullrequest集合
• 长轮询,PullRequestHoldService会每隔5秒重新执行挂起的pullrequest集合
• 长轮询15秒超时,最多挂起3次;短轮询1秒超时,最多挂起一次
• reputMessageService当处理新消息会调用MessageArriving唤醒PullRequestHoldService处理挂起的请求

 
/**
 * reput message service 在处理consumequeue和index后会触发
 * messagearrivingListener.arriving 触发holdservice处理
 */
public class PullRequestHoldService extends ServiceThread {
    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
    private static final String TOPIC_QUEUEID_SEPARATOR = "@";
    private final BrokerController brokerController;
    private final SystemClock systemClock = new SystemClock();
    /**
     * 长轮询的请求集合
     */
    private ConcurrentMap<String/* topic@queueId */, ManyPullRequest/*没有拉取到消息的请求集合*/> pullRequestTable =
        new ConcurrentHashMap<String, ManyPullRequest>(1024);

    public PullRequestHoldService(final BrokerController brokerController) {
        this.brokerController = brokerController;
    }

    public void suspendPullRequest(final String topic, final int queueId, final PullRequest pullRequest) {
        String key = this.buildKey(topic, queueId);
        ManyPullRequest mpr = this.pullRequestTable.get(key);
        if (null == mpr) {
            mpr = new ManyPullRequest();
            ManyPullRequest prev = this.pullRequestTable.putIfAbsent(key, mpr);
            if (prev != null) {
                mpr = prev;
            }
        }

        mpr.addPullRequest(pullRequest);
    }

    private String buildKey(final String topic, final int queueId) {
        StringBuilder sb = new StringBuilder();
        sb.append(topic);
        sb.append(TOPIC_QUEUEID_SEPARATOR);
        sb.append(queueId);
        return sb.toString();
    }

    @Override
    public void run() {
        // PullRequestHoldService线程自旋处理
        log.info("{} service started", this.getServiceName());
        while (!this.isStopped()) {
            try {
                // broker如果是长轮询则5秒重试
                if (this.brokerController.getBrokerConfig().isLongPollingEnable()) {
                    this.waitForRunning(5 * 1000);
                } else {
                    // broker如果是短轮询则1秒重试
                    this.waitForRunning(this.brokerController.getBrokerConfig().getShortPollingTimeMills());
                }

                long beginLockTimestamp = this.systemClock.now();
                // 核心逻辑
                this.checkHoldRequest();
                long costTime = this.systemClock.now() - beginLockTimestamp;
                if (costTime > 5 * 1000) {
                    log.info("[NOTIFYME] check hold request cost {} ms.", costTime);
                }
            } catch (Throwable e) {
                log.warn(this.getServiceName() + " service has exception. ", e);
            }
        }

        log.info("{} service end", this.getServiceName());
    }

    @Override
    public String getServiceName() {
        return PullRequestHoldService.class.getSimpleName();
    }

    private void checkHoldRequest() {
        for (String key : this.pullRequestTable.keySet()) {
            String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR);
            if (2 == kArray.length) {
                String topic = kArray[0];
                int queueId = Integer.parseInt(kArray[1]);
                //  根据主题，消费队列ID查找队列的最大偏移量
                final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
                try {
                    // 根据该offset，判断是否有新的消息达到。有则执行信息拉取
                    this.notifyMessageArriving(topic, queueId, offset);
                } catch (Throwable e) {
                    log.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
                }
            }
        }
    }

    public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) {
        notifyMessageArriving(topic, queueId, maxOffset, null, 0, null, null);
    }

    /**
     * reputmessageService 触发
     * @param topic
     * @param queueId
     * @param maxOffset 消费队列当前最大偏移量
     * @param tagsCode 消息tag hashcode //基于tag消息过滤
     * @param msgStoreTime
     * @param filterBitMap 过滤位图。
     * @param properties 消息属性,基于属性的消息过滤。
     */
    public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode,
        long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
        String key = this.buildKey(topic, queueId);
        ManyPullRequest mpr = this.pullRequestTable.get(key);
        if (mpr != null) {
            // 获取主题与队列的所有 PullRequest 并清除内部 pullRequest 集合，避免重复拉取
            List<PullRequest> requestList = mpr.cloneListAndClear();
            if (requestList != null) {
                List<PullRequest> replayList = new ArrayList<PullRequest>();

                for (PullRequest request : requestList) {
                    long newestOffset = maxOffset;
                    // 如果待拉取偏移量(pullFromThisOffset)小于消息队列的最大有效偏移量，则再次获取消息队列的最大有效偏移量，再给一次机会
                    if (newestOffset <= request.getPullFromThisOffset()) {
                        newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
                    }
                    // 如果队列最大偏移量大于 pullFromThisOffset 说明有新的消息到达，先简单对消息根据 tag,属性进行一次消息过滤，如果 tag,属性为空，则消息过滤器会返回true,然后 executeRequestWhenWakeup进行消息拉取，结束长轮询
                    if (newestOffset > request.getPullFromThisOffset()) {
                        boolean match = request.getMessageFilter().isMatchedByConsumeQueue(tagsCode,
                            new ConsumeQueueExt.CqExtUnit(tagsCode, msgStoreTime, filterBitMap));
                        // match by bit map, need eval again when properties is not null.
                        if (match && properties != null) {
                            match = request.getMessageFilter().isMatchedByCommitLog(null, properties);
                        }

                        if (match) {
                            try {
                                // 同消息拉取进程
                                this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
                                    request.getRequestCommand());
                            } catch (Throwable e) {
                                log.error("execute request when wakeup failed.", e);
                            }
                            continue;
                        }
                    }
                    // 如果挂起时间超过 suspendTimeoutMillisLong，则超时，结束长轮询，调用executeRequestWhenWakeup 进行消息拉取，并返回结果到客户端
                    if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request.getTimeoutMillis())) {
                        try {
                            // 此时brokerAllowSuspend=false 也就是不允许挂起 需要返回客户端
                            this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
                                request.getRequestCommand());
                        } catch (Throwable e) {
                            log.error("execute request when wakeup failed.", e);
                        }
                        continue;
                    }

                    replayList.add(request);
                }

                if (!replayList.isEmpty()) {
                    mpr.addPullRequest(replayList);
                }
            }
        }
    }
}