浅析RocketMQ-发个消息

本文将解析生产者发送消息到broker的流程。

DefaultMQProducer调用send发送消息,最终实现是DefaultMQProducerImpl#sendDefaultImpl。
sendDefaultImpl主要有4步流程:

1. 获取主题路由信息
2. 负载选择合适的消息队列
3. 发送消息到broker
4. 如果开启容错机制，更新容错队列

 private SendResult sendDefaultImpl(
        Message msg,
        final CommunicationMode communicationMode,
        final SendCallback sendCallback,
        final long timeout
    ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
        // 省略校验
        final long invokeID = random.nextLong();
        // 几个时间用于记录执行时间，以及容错机制记录使用
        long beginTimestampFirst = System.currentTimeMillis();
        long beginTimestampPrev = beginTimestampFirst;
        long endTimestamp = beginTimestampFirst;
        // 1. 获取主题发布信息
        TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic());
        if (topicPublishInfo != null && topicPublishInfo.ok()) {
            boolean callTimeout = false;
            MessageQueue mq = null;
            Exception exception = null;
            SendResult sendResult = null;
            // 重试次数，异步只执行一次
            int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1;
            int times = 0;
            String[] brokersSent = new String[timesTotal];
            for (; times < timesTotal; times++) {
                String lastBrokerName = null == mq ? null : mq.getBrokerName();
                // 2. 负载均衡选择消息队列
                MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName);
                if (mqSelected != null) {
                    mq = mqSelected;
                    brokersSent[times] = mq.getBrokerName();
                    try {
                        // ...省略不重要代码
						// 3. 发送消息，
                        sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime);
                        endTimestamp = System.currentTimeMillis();
                        // 4. 如果开启容错，更新容错队列
                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false);
                        // 异步以及单向 返回null，同步发送返回结果
                        switch (communicationMode) {
                            case ASYNC:
                                return null;
                            case ONEWAY:
                                return null;
                            case SYNC:
                                if (sendResult.getSendStatus() != SendStatus.SEND_OK) {
                                    if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) {
                                        continue;
                                    }
                                }

                                return sendResult;
                            default:
                                break;
                        }
                    } catch (RemotingException e) {
                        endTimestamp = System.currentTimeMillis();
                        // 如果开启容错，更新容错队列
                        this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true);
                        continue;
                    } 
                    // 省略其他类型的异常处理,除了InterruptedException中断重试，其他类型异常继续重试
                }
            }
			// 省略异常处理代码
        }
    }

一. 获取主题路由信息

tryToFindTopicPublishInfo流程很简单：

1. 先从本地查找
2. 本地存在从nameServer查找
3. nameServer也不存在，则使用默认主题的数据

    private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) {
    	// 先从本地路由缓存中查询
        TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic);
        // 本地路由缓存不存在
        if (null == topicPublishInfo || !topicPublishInfo.ok()) {
            this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo());
            // 尝试从nameServer获取路由数据
            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
            topicPublishInfo = this.topicPublishInfoTable.get(topic);
        }

        if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) {
            return topicPublishInfo;
        } else {
        	// nameServer获取不到对应topic数据，则使用默认主题，进行发送消息
            this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer);
            topicPublishInfo = this.topicPublishInfoTable.get(topic);
            return topicPublishInfo;
        }
    }

接着进入updateTopicRouteInfoFromNameServer

    public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault,
        DefaultMQProducer defaultMQProducer) {
            if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
                try {
                    TopicRouteData topicRouteData;
                    // 默认主题获取路由数据
                    if (isDefault && defaultMQProducer != null) {
                        topicRouteData = this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.getCreateTopicKey(),
                            clientConfig.getMqClientApiTimeout());
                        if (topicRouteData != null) {
                            for (QueueData data : topicRouteData.getQueueDatas()) {
                                int queueNums = Math.min(defaultMQProducer.getDefaultTopicQueueNums(), data.getReadQueueNums());
                                data.setReadQueueNums(queueNums);
                                data.setWriteQueueNums(queueNums);
                            }
                        }
                    } else {
                    // 一般主题获取路由数据
                        topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout());
                    }
                    if (topicRouteData != null) {
                        TopicRouteData old = this.topicRouteTable.get(topic);
                        // 判断路由数据是否需要更新
                        boolean changed = topicRouteDataIsChange(old, topicRouteData);
                        if (!changed) {
                            changed = this.isNeedUpdateTopicRouteInfo(topic);
                        } 
                        if (changed) {
                            TopicRouteData cloneTopicRouteData = topicRouteData.cloneTopicRouteData();
							// 更新本地broker地址数据
                            for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                                this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());
                            }

                            //  生产者路由数据更新
                            if (!producerTable.isEmpty()) {
                            	// 获取写队列，组装MessageQueue数据，如果是有序的，则按照指定顺序组装
                                TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData);
                                publishInfo.setHaveTopicRouterInfo(true);
                                // 获取生产者组下的每个生产者，并更新对应的路由数据	
                                Iterator<Entry<String, MQProducerInner>> it = this.producerTable.entrySet().iterator();
                                while (it.hasNext()) {
                                    Entry<String, MQProducerInner> entry = it.next();
                                    MQProducerInner impl = entry.getValue();
                                    if (impl != null) {
                                    	// 将topic以及路由信息存入topicPublishInfoTable
                                        impl.updateTopicPublishInfo(topic, publishInfo);
                                    }
                                }
                            }

                            //  消费者路由数据更新
                            if (!consumerTable.isEmpty()) {
                            // 获取读队列，组装MessageQueue数据
                                Set<MessageQueue> subscribeInfo = topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
                                // 获取生产者组下的每个生产者，并更新对应的路由数据	
                                Iterator<Entry<String, MQConsumerInner>> it = this.consumerTable.entrySet().iterator();
                                while (it.hasNext()) {
                                    Entry<String, MQConsumerInner> entry = it.next();
                                    MQConsumerInner impl = entry.getValue();
                                    if (impl != null) {
                                        impl.updateTopicSubscribeInfo(topic, subscribeInfo);
                                    }
                                }
                            }
                            this.topicRouteTable.put(topic, cloneTopicRouteData);
                            return true;
                        }
                    } 
                    // ,,, 省略不相关代码
                } 
            } 
        return false;
    }

getTopicRouteInfoFromNameServer 方法会向namesever拉取TopicRouteData 数据，namesever具体调佣RouteInfoManager#pickupTopicRouteData。

public class TopicRouteData extends RemotingSerializable {
	// 有序性配置数据,大体结构： brokerName:queueId;brokerName:queueId
    private String orderTopicConf;
    // topic关联的各个队列数据，QueueData包含每个broker对于该主题的配置数据
    private List<QueueData> queueDatas;
    // topic关联的各个broker数据，BrokerData包含broker的集群名，broker名，broker地址列表
    private List<BrokerData> brokerDatas;
    // 每个broker对应过滤器列表
    private HashMap<String/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
}

执行 topicRouteData2TopicPublishInfo会将TopicRouteData 里面的queueDatas转化为messageQueueList。
假如topic-a主题对应两个broker，且每个broker都包含4个写队列，则转为为messageQueueList后有8个有序的MessageQueue。

二. 负载均衡

rocketMQ负载都是在client端完成，可分为Producer负载和Consumer负载。这里看下Producer的负载处理。

    public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
    	// 判断是否开启容错机制
        if (this.sendLatencyFaultEnable) {
        		// 随机挑选一个broker下面的队列，逻辑与selectOneMessageQueue类似
                int index = tpInfo.getSendWhichQueue().incrementAndGet();
                for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) {
                    int pos = Math.abs(index++) % tpInfo.getMessageQueueList().size();
                    if (pos < 0)
                        pos = 0;
                    MessageQueue mq = tpInfo.getMessageQueueList().get(pos);
                    // mq对应的broker，不在容错队列 || 当前broker可用，则返回
                    if (latencyFaultTolerance.isAvailable(mq.getBrokerName()))
                        return mq;
                }
				// 循环了所有队列，都是有问题的，则挑选一个相对可用的broker
                final String notBestBroker = latencyFaultTolerance.pickOneAtLeast();
                // 找出对应可写队列
                int writeQueueNums = tpInfo.getQueueIdByBroker(notBestBroker);
                if (writeQueueNums > 0) {
                    final MessageQueue mq = tpInfo.selectOneMessageQueue();
                    if (notBestBroker != null) {
                        mq.setBrokerName(notBestBroker);
                        mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums);
                    }
                    return mq;
                } else {
                	// 没有可写队列，直接移除对应broker
                    latencyFaultTolerance.remove(notBestBroker);
                }
            // 没找到合适随机挑一个    
            return tpInfo.selectOneMessageQueue();
        }
		// 未开启容错，随机获取一个消息队列
        return tpInfo.selectOneMessageQueue(lastBrokerName);
    }

1. selectOneMessageQueue

selectOneMessageQueue会随机挑选一个消息队列，如果之前消息队列出问题了，此次会换一个broker的队列

    public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
    	// 首次发送消息
        if (lastBrokerName == null) {
        	// 随机抽取一个幸运的队列
            return selectOneMessageQueue();
        } else {
        	// 之前发送异常了，挑选下一个队列
            for (int i = 0; i < this.messageQueueList.size(); i++) {
                int index = this.sendWhichQueue.incrementAndGet();
                int pos = Math.abs(index) % this.messageQueueList.size();
                if (pos < 0)
                    pos = 0;
                MessageQueue mq = this.messageQueueList.get(pos);
               	// 如果选中的队列broker和之前队列的broker不是同一个直接返回
                if (!mq.getBrokerName().equals(lastBrokerName)) {
                    return mq;
                }
            }
            // 是在选不到合适的，随机挑个
            return selectOneMessageQueue();
        }
    }

2. pickOneAtLeast

pickOneAtLeast用于挑选一个相对可用的broker

    public String pickOneAtLeast() {
        final Enumeration<FaultItem> elements = this.faultItemTable.elements();
        List<FaultItem> tmpList = new LinkedList<FaultItem>();

        while (elements.hasMoreElements()) {
            final FaultItem faultItem = elements.nextElement();
            tmpList.add(faultItem);
        }
        if (!tmpList.isEmpty()) {
        	// 打乱顺序
            Collections.shuffle(tmpList);
			// 排序，可用的broker排前面
			// 排序规则: 1.broker可用 2.延迟小 3.可解除延迟时间小的
            Collections.sort(tmpList);
			// 这里取一半，后面的都是问题比较严重的，不进行考虑
            final int half = tmpList.size() / 2;
            if (half <= 0) {
                return tmpList.get(0).getName();
            } else {
                final int i = this.whichItemWorst.incrementAndGet() % half;
                return tmpList.get(i).getName();
            }
        }

        return null;
    }

三. 发送消息

sendKernelImpl 主要进行数据组装，执行hook操作，发送消息到broker。

private SendResult sendKernelImpl(final Message msg,
        final MessageQueue mq,
        final CommunicationMode communicationMode,
        final SendCallback sendCallback,
        final TopicPublishInfo topicPublishInfo,
        final long timeout) {
        long beginStartTime = System.currentTimeMillis();
        // 本地缓存获取broker地址
        String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
        if (null == brokerAddr) {
        	// broker不存在，去线上拉取，更新一遍本地缓存
            tryToFindTopicPublishInfo(mq.getTopic());
            brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
        }

        SendMessageContext context = null;
        if (brokerAddr != null) {
            brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr);

            byte[] prevBody = msg.getBody();
               	// ... 
               	// 发送数据大于4k,进行压缩
                if (this.tryToCompressMessage(msg)) {
                    sysFlag |= MessageSysFlag.COMPRESSED_FLAG;
                    msgBodyCompressed = true;
                }

                // ...
				// hook操作
                if (hasCheckForbiddenHook()) {
					...
                }
                if (this.hasSendMessageHook()) {
                   ...
                }

                SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
                // 省略组装requestHeader 数据

                SendResult sendResult = null;
                // 发送请求
                switch (communicationMode) {
                    case ASYNC:
                        ...
                        sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(...);
                        break;
                    case ONEWAY:
                    case SYNC:
                        long costTimeSync = System.currentTimeMillis() - beginStartTime;
                        if (timeout < costTimeSync) {
                            throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
                        }
                        sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(...);
                        break;
                    default:
                        break;
                }
				// 返回结果回调处理
                if (this.hasSendMessageHook()) {
                    context.setSendResult(sendResult);
                    this.executeSendMessageHookAfter(context);
                }

                return sendResult;
        }
		... 
    }

四. 更新容错队列

发送消息完成，如果开启容错机制，则更新容错队列。currentLatency为执行耗时,isolation这里为false

   public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) {
   		// 开启容错
        if (this.sendLatencyFaultEnable) {
        	// isolation这里为false，则采用发送消息的耗时，否则默认3s
            long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency);
            // 更新容错队列
            this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration);
        }
    }

    private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L};
    private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L};

    private long computeNotAvailableDuration(final long currentLatency) {
    	// 确认延迟级别，返回不可用时间
        for (int i = latencyMax.length - 1; i >= 0; i--) {
            if (currentLatency >= latencyMax[i])
                return this.notAvailableDuration[i];
        }

        return 0;
    }