目录：
- RocketMQ源码解析——搭建源码环境
- RocketMQ源码解析——NameServer
- RocketMQ源码解析——Broker
- RocketMQ源码解析——Producer
- RocketMQ源码解析——消息存储
- RocketMQ源码解析——Consumer
1. 了解消息生产者——Producer：
RocketMQ中的消息生产者，主要负责生产消息、从NameServer中获取最新的Broker信息，以及将消息发送到Broker
那么在源码中，消息生产者代码都在client包中，因为对于RocketMQ来说，消息生产者就是客户端。而最常用的消息生产者类是DefaultMQProducer：
从上图可以了解到DefaultMQProducer的上层的继承体系，其中实现两个核心的接口类。为了对DefaultMQProducer这个消息生产者类有深入的了解，先来看下他的接口父类有哪些重要的接口和属性：
- MQAdmin：
public interface MQAdmin {
/**
* 创建主题
*/
void createTopic(final String key, final String newTopic, final int queueNum) throws MQClientException;
void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException;
/**
* 根据时间戳从队列中查找消息偏移量
*/
long searchOffset(final MessageQueue mq, final long timestamp) throws MQClientException;
/**
* 查找消息队列中最大偏移量
*/
long maxOffset(final MessageQueue mq) throws MQClientException;
/**
* 查找消息队列中最小的偏移量
*/
long minOffset(final MessageQueue mq) throws MQClientException;
/**
* 查找消息队列中最早的存储消息时间
*/
long earliestMsgStoreTime(final MessageQueue mq) throws MQClientException;
/**
* 根据消息id查找消息
*/
MessageExt viewMessage(final String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException;
/**
* 根据条件查找消息
*/
QueryResult queryMessage(final String topic, final String key, final int maxNum, final long begin, final long end) throws MQClientException, InterruptedException;
/**
* 根据主题和消息id查找消息
*/
MessageExt viewMessage(String topic, String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException;
}
- MQProducer：
public interface MQProducer extends MQAdmin {
/**
* 启动生产者
*/
void start() throws MQClientException;
/**
* 关闭生产者
*/
void shutdown();
/**
* 查找指定主题下的所有消息
*/
List<MessageQueue> fetchPublishMessageQueues(final String topic) throws MQClientException;
/**
* 发送同步消息
*/
SendResult send(final Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Message msg, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Message msg, final MessageQueue mq) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Message msg, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
/**
* 发送异步消息
*/
void send(final Message msg, final SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException;
void send(final Message msg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException;
void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException;
void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, long timeout) throws MQClientException, RemotingException, InterruptedException;
void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException;
void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException;
/**
* 发送单向消息
*/
void sendOneway(final Message msg) throws MQClientException, RemotingException, InterruptedException;
void sendOneway(final Message msg, final MessageQueue mq) throws MQClientException, RemotingException, InterruptedException;
void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg) throws MQClientException, RemotingException, InterruptedException;
/**
* 发送事务消息
*/
TransactionSendResult sendMessageInTransaction(final Message msg, final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException;
TransactionSendResult sendMessageInTransaction(final Message msg, final Object arg) throws MQClientException;
/**
* 批量发送消息
*/
SendResult send(final Collection<Message> msgs) throws MQClientException, RemotingException, MQBrokerException,
InterruptedException;
SendResult send(final Collection<Message> msgs, final long timeout) throws MQClientException,
RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Collection<Message> msgs, final MessageQueue mq) throws MQClientException,
RemotingException, MQBrokerException, InterruptedException;
SendResult send(final Collection<Message> msgs, final MessageQueue mq, final long timeout)
throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
void send(final Collection<Message> msgs, final SendCallback sendCallback) throws MQClientException, RemotingException, MQBrokerException,
InterruptedException;
void send(final Collection<Message> msgs, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException,
MQBrokerException, InterruptedException;
void send(final Collection<Message> msgs, final MessageQueue mq, final SendCallback sendCallback) throws MQClientException, RemotingException,
MQBrokerException, InterruptedException;
void send(final Collection<Message> msgs, final MessageQueue mq, final SendCallback sendCallback, final long timeout) throws MQClientException,
RemotingException, MQBrokerException, InterruptedException;
//for rpc
Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException,
RemotingException, MQBrokerException, InterruptedException;
void request(final Message msg, final RequestCallback requestCallback, final long timeout)
throws MQClientException, RemotingException, InterruptedException, MQBrokerException;
Message request(final Message msg, final MessageQueueSelector selector, final Object arg,
final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException,
InterruptedException;
void request(final Message msg, final MessageQueueSelector selector, final Object arg,
final RequestCallback requestCallback,
final long timeout) throws MQClientException, RemotingException,
InterruptedException, MQBrokerException;
Message request(final Message msg, final MessageQueue mq, final long timeout)
throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException;
void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout)
throws MQClientException, RemotingException, MQBrokerException, InterruptedException;
}
- DefaultMQProducer：
protected final transient DefaultMQProducerImpl defaultMQProducerImpl; // 生产者包装类，实际工作的是该实例对象
private String producerGroup; // 消息组
private String createTopicKey = TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC; // 默认topic
private volatile int defaultTopicQueueNums = 4; // 默认每个主题在Broker中的数量
private int sendMsgTimeout = 3000; // 发送消息超时时间
private int compressMsgBodyOverHowmuch = 1024 * 4; // 单次发送消息总体积大小（默认4k），超过需要进行压缩
private int retryTimesWhenSendFailed = 2; // 同步发送失败重试次数
private int retryTimesWhenSendAsyncFailed = 2; // 异步发送失败重试次数
private boolean retryAnotherBrokerWhenNotStoreOK = false; // 当发送消息失败时，是否选择另一个Broker发送消息，默认为false
private int maxMessageSize = 1024 * 1024 * 4; // 允许发送消息的最大体积
private TraceDispatcher traceDispatcher = null; // 异步消息发送的请求分发器
2. 源码分析：
上一节了解了生产者重点类的属性和方法，这一节就开始针对源码进行分析
为了能够更直观的理解源码，首先熟悉下生产者整个启动流程，如下图所示：
- 启动生产者并发送消息：
public class Producer {
public static void main(String[] args) throws MQClientException, InterruptedException {
DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); // 创建生产者，并指定组名
producer.start(); // 启动生产者
for (int i = 0; i < 1000; i++) { // 发送1000条消息
try {
Message msg = new Message("TopicTest" /* Topic */,
"TagA" /* Tag */,
("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */
); // 创建消息
SendResult sendResult = producer.send(msg); // 发送同步消息
System.out.printf("%s%n", sendResult);
} catch (Exception e) {
e.printStackTrace();
Thread.sleep(1000);
}
}
producer.shutdown(); // 消息发送完毕，关闭生产者
}
}
- 启动生产者：
public void start() throws MQClientException {
this.setProducerGroup(withNamespace(this.producerGroup)); // 设置组名
this.defaultMQProducerImpl.start(); // 启动生产者实例
if (null != traceDispatcher) {
try {
traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel());
} catch (MQClientException e) {
log.warn("trace dispatcher start failed ", e);
}
}
}
public void start() throws MQClientException {
this.start(true);
}
public void start(final boolean startFactory) throws MQClientException {
switch (this.serviceState) { // 状态默认为CREATE_JUST，表示服务刚创建，尚未启动
case CREATE_JUST:
this.serviceState = ServiceState.START_FAILED; // 首次标识为启动失败
this.checkConfig(); // 检查生产者配置
if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { // 组名不等于RocketMQ内部已定义组名
this.defaultMQProducer.changeInstanceNameToPID(); // 更改当前instanceName为进程ID
}
this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); // 创建MQ客户端实例
boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); // 注册当前生产者
if (!registerOK) { // 注册失败
this.serviceState = ServiceState.CREATE_JUST; // 回复初始状态
throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null);
}
// 注册成功
this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); // 保存消息的topic信息
if (startFactory) {
mQClientFactory.start();
}
log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), this.defaultMQProducer.isSendMessageWithVIPChannel());
this.serviceState = ServiceState.RUNNING; // 标识生产者状态为已启动
break;
// 其他状态在启动过程中都视为非法状态
case RUNNING:
case START_FAILED:
case SHUTDOWN_ALREADY:
throw new MQClientException("The producer service state not OK, maybe started once, " + this.serviceState + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null);
default:
break;
}
this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); // 发送心跳消息给所有Broker
this.timer.scheduleAtFixedRate(new TimerTask() { // 计划定时任务，首次执行任务前延时3秒，之后每隔1秒执行任务
@Override
public void run() {
try {
RequestFutureTable.scanExpiredRequest(); // 扫描过期请求，并移除
} catch (Throwable e) {
log.error("scan RequestFutureTable exception", e);
}
}
}, 1000 * 3, 1000);
}
- 检查生产者配置信息：
private void checkConfig() throws MQClientException {
Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); // 校验组名
if (null == this.defaultMQProducer.getProducerGroup()) { // 组名不能为空
throw new MQClientException("producerGroup is null", null);
}
if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { // 组名不能与RocketMQ内部默认的组名相等
throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", null);
}
}
public static void checkGroup(String group) throws MQClientException {
if (UtilAll.isBlank(group)) { // 不能为空
throw new MQClientException("the specified group is blank", null);
}
if (group.length() > CHARACTER_MAX_LENGTH) { // 组名长度不能超过限定长度
throw new MQClientException("the specified group is longer than group max length 255.", null);
}
if (!regularExpressionMatcher(group, PATTERN)) { // 组名需要匹配对应的字符模式
throw new MQClientException(String.format("the specified group[%s] contains illegal characters, allowing only %s", group, VALID_PATTERN_STR), null);
}
}
- 更改当前instanceName为进程ID：
public void changeInstanceNameToPID() {
if (this.instanceName.equals("DEFAULT")) { // 实例名等于DEFAULT
this.instanceName = UtilAll.getPid() + "#" + System.nanoTime(); // 将实例名改为 进程名 + # + 系统当前时间毫秒值
}
}
- 注册消息生产者：
public boolean registerProducer(final String group, final DefaultMQProducerImpl producer) {
if (null == group || null == producer) {
return false;
}
MQProducerInner prev = this.producerTable.putIfAbsent(group, producer); // 将<组名, 消息生产者实例>信息保存进生产者映射表中
if (prev != null) {
log.warn("the producer group[{}] exist already.", group);
return false;
}
return true;
}
在以上启动生产者的源码分析的过程中涉及到一个贯穿整个生产者生命周期的重点类——TopicPublishInfo，需要重点关注：
public class TopicPublishInfo {
private boolean orderTopic = false; // 是否是顺序消息，默认为false
private boolean haveTopicRouterInfo = false;
private List<MessageQueue> messageQueueList = new ArrayList<MessageQueue>(); // 该主题的消息队列
private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); // 每选择一次消息队列，该值加一
private TopicRouteData topicRouteData; // 关联的topic路由消息
}
以上源码分析便是消息生产者的启动过程
消息生产者启动后，下一步便是发送消息。还是为了更好的理解之后的源码解析，首先了解发送消息的整个执行过程，如下图所示：
- 发送消息：
public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
Validators.checkMessage(msg, this); // 检查消息是否合法
msg.setTopic(withNamespace(msg.getTopic()));
return this.defaultMQProducerImpl.send(msg); // 发送消息
}
- 检查消息是否合法：
public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer) throws MQClientException {
if (null == msg) {
throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message is null");
}
// 检查消息的topic是否合法
Validators.checkTopic(msg.getTopic());
Validators.isNotAllowedSendTopic(msg.getTopic());
// 消息体不能为空 && 消息长度不能为0 && 消息体长度不能大于规定最大长度（4M）
if (null == msg.getBody()) {
throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body is null");
}
if (0 == msg.getBody().length) {
throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body length is zero");
}
if (msg.getBody().length > defaultMQProducer.getMaxMessageSize()) {
throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body size over max value, MAX: " + defaultMQProducer.getMaxMessageSize());
}
}
- 生产者实例发送同步消息：
public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
return send(msg, this.defaultMQProducer.getSendMsgTimeout()); // 发送同步消息，并设有发送超时时间
}
public SendResult send(Message msg, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout);
}
private SendResult sendDefaultImpl(Message msg, final CommunicationMode communicationMode, final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
this.makeSureStateOK(); // 校验生产者状态，确保处于RUNNING正在运行状态
Validators.checkMessage(msg, this.defaultMQProducer); // 再次校验消息是否合法
final long invokeID = random.nextLong();
long beginTimestampFirst = System.currentTimeMillis();
long beginTimestampPrev = beginTimestampFirst;
long endTimestamp = beginTimestampFirst;
TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); // 根据消息的主题获取该主题相关信息
if (topicPublishInfo != null && topicPublishInfo.ok()) {
boolean callTimeout = false;
MessageQueue mq = null;
Exception exception = null;
SendResult sendResult = null;
int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; // 计算最大可重试次数（同步发送默认最多可重试3次）
int times = 0;
String[] brokersSent = new String[timesTotal];
for (; times < timesTotal; times++) { // 在最大可重试次数的范围中发送消息
String lastBrokerName = null == mq ? null : mq.getBrokerName();
MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); // 根据路由信息和上一次选择的Broker名，选择消息队列
if (mqSelected != null) {
mq = mqSelected;
brokersSent[times] = mq.getBrokerName(); // 记录此次选择接受消息的Broke名
try {
beginTimestampPrev = System.currentTimeMillis();
if (times > 0) { // 重发次数大于0，则说明存在发送消息失败的情况
msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic())); // 重置发送的消息主题
}
long costTime = beginTimestampPrev - beginTimestampFirst;
if (timeout < costTime) { // 超时，直接跳出循环，抛异常
callTimeout = true;
break;
}
sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); // 发送消息
endTimestamp = System.currentTimeMillis();
this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); //
switch (communicationMode) {
case ASYNC: // 异步发送消息，不需要返回值
return null;
case ONEWAY: // 单向发送消息，不需要返回值
return null;
case SYNC: // 同步发送消息，需要返回值
if (sendResult.getSendStatus() != SendStatus.SEND_OK) { // 消息发送结果失败
if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { // 允许重试
continue; // 继续重试消息发送
}
}
return sendResult; // 否则，返回消息发送成功响应结果
default:
break;
}
} catch (RemotingException e) {
...
} catch (MQClientException e) {
...
} catch (MQBrokerException e) {
...
} catch (InterruptedException e) {
...
}
} else {
break;
}
}
if (sendResult != null) { // 返回消息发送失败响应结果
return sendResult;
}
String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s",
times,
System.currentTimeMillis() - beginTimestampFirst,
msg.getTopic(),
Arrays.toString(brokersSent));
info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED);
MQClientException mqClientException = new MQClientException(info, exception);
if (callTimeout) { // 超时，抛异常
throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout");
}
if (exception instanceof MQBrokerException) {
mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode());
} else if (exception instanceof RemotingConnectException) {
mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION);
} else if (exception instanceof RemotingTimeoutException) {
mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT);
} else if (exception instanceof MQClientException) {
mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION);
}
throw mqClientException;
}
validateNameServerSetting();
throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO),
null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION);
}
- 根据消息主题获取该主题相关信息：
private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) {
TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); // 从主题映射表中根据主题名获取主题相关信息
if (null == topicPublishInfo || !topicPublishInfo.ok()) { // 主题信息为空，或者属于非法状态
this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); // 添加新主题
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); // 同时将主题信息同步更新到NameServer的路由信息中
topicPublishInfo = this.topicPublishInfoTable.get(topic); // 再次获取主题
}
if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { // 再次校验主题信息，主题信息中包含路由信息，以及主题状态合法
return topicPublishInfo; // 直接返回主题信息
} else {
// 若未找到当前主题的路由信息,则用默认主题继续查找
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer);
topicPublishInfo = this.topicPublishInfoTable.get(topic);
return topicPublishInfo;
}
}
- 判断主题信息状态：
public boolean ok() {
return null != this.messageQueueList && !this.messageQueueList.isEmpty(); // 其实就是判断主题信息中包含的消息队列是否为空
}
- 选择消息发送队列：
public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName);
}
在选择消息发送队列时候，有两种策略：
1. 开启故障延时机制策略
2. 关闭故障延时机制策略（默认）
因为在生产者发送消息的时候，可能因为网络原因或者是其他外接因素，导致消息发送失败，这个时候就会进行消息发送的重试
所谓开启故障延时机制，就是会将这个接受消息失败的Broker记录下来，等到下次轮询到这个Broker的时候就会跳过这个Broker，这样就减少了消息发送失败的概率
而关闭故障延时机制则不会理会Broker是否有过故障，选择后直接发送消息
那么先来了解在开启故障延时机制下是如何选择消息发送队列：
public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) {
if (this.sendLatencyFaultEnable) { // 启用故障延时机制
try {
int index = tpInfo.getSendWhichQueue().incrementAndGet(); // index自增1
for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { // 遍历主题中包含的所有队列
// 轮询获取队列
int pos = Math.abs(index++) % tpInfo.getMessageQueueList().size();
if (pos < 0)
pos = 0;
MessageQueue mq = tpInfo.getMessageQueueList().get(pos);
if (latencyFaultTolerance.isAvailable(mq.getBrokerName())) // 校验队列对应的Broker是否可用
return mq; // 可用，返回队列
}
// 以上所有队列对应的Broker都判断为不可用，那么没办法，也只能强行选择一个Broker
final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); // 选择一个非最优的Broker名
int writeQueueNums = tpInfo.getQueueIdByBroker(notBestBroker); // 获取该Broker中的写队列数量
if (writeQueueNums > 0) { // 写队列数量大于0，说明可以接受消息
final MessageQueue mq = tpInfo.selectOneMessageQueue(); // 选择一个队列，重新设置Broker和队列id，并返回
if (notBestBroker != null) {
mq.setBrokerName(notBestBroker);
mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums);
}
return mq;
} else { // 写队列数量小于0，说明已无法接收消息，直接移除该Broker
latencyFaultTolerance.remove(notBestBroker);
}
} catch (Exception e) {
log.error("Error occurred when selecting message queue", e);
}
return tpInfo.selectOneMessageQueue(); // 经过上面的筛选，还是没有找到合适的Broker，则直接选择一个队列
}
return tpInfo.selectOneMessageQueue(lastBrokerName); // 非故障延时机制
}
从以上基于故障延时机制策略选择队列的源码中，发现其中依赖一个核心接口类——LatencyFaultTolerance：
public interface LatencyFaultTolerance<T> {
void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); // 更新接受消息失败的Broker条目
boolean isAvailable(final T name); // 判断Broker是否可用
void remove(final T name); // 移除接受消息失败的Broker条目
T pickOneAtLeast(); // 从曾接收消息失败的Broker中选择一个可用的Broker
}
既然是接口类，那么必然有实现类——LatencyFaultToleranceImpl：
public class LatencyFaultToleranceImpl implements LatencyFaultTolerance<String> {
private final ConcurrentHashMap<String, FaultItem> faultItemTable = new ConcurrentHashMap<String, FaultItem>(16); // 失败条目映射表，记录接收消息失败的Broker信息（实际上保存的是<Broker名，接受消息失败的Broker条目>）
private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex();
/**
* 更新失败条目映射表
*/
@Override
public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) {
FaultItem old = this.faultItemTable.get(name); // 根据Broker名获取Broker
if (null == old) { // 为空，说明该Broker是首次接收消息失败
// 记录该Broker，将其保存进失败条目映射表中
final FaultItem faultItem = new FaultItem(name);
faultItem.setCurrentLatency(currentLatency);
faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
old = this.faultItemTable.putIfAbsent(name, faultItem);
if (old != null) {
old.setCurrentLatency(currentLatency);
old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
}
} else { // 不为空，说明该Broker再次接收消息失败，更新其 本次消息发送延时时间 和 故障规避时间（即下次允许发送消息时间戳）
old.setCurrentLatency(currentLatency);
old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration);
}
}
/**
* 判断Broker是否可用
*/
@Override
public boolean isAvailable(final String name) {
final FaultItem faultItem = this.faultItemTable.get(name);
if (faultItem != null) { // 存在失败条目Broker
return faultItem.isAvailable(); // 判断Broker是否可用（即判断当前系统时间戳是否大于故障规避时间戳）
}
return true; // 不存在失败条目Broker，说明指定name的Broker未曾接受消息失败，可用
}
/**
* 从失败条目映射表中移除Broker
*/
@Override
public void remove(final String name) {
this.faultItemTable.remove(name);
}
/**
* 从失败条目映射表中选择一个故障延时时间戳较前的Broker名
*/
@Override
public String pickOneAtLeast() {
// 将失败条目都复制添加到tmpList中
final Enumeration<FaultItem> elements = this.faultItemTable.elements();
List<FaultItem> tmpList = new LinkedList<FaultItem>();
while (elements.hasMoreElements()) {
final FaultItem faultItem = elements.nextElement();
tmpList.add(faultItem);
}
if (!tmpList.isEmpty()) {
Collections.shuffle(tmpList);
Collections.sort(tmpList); // 排序（故障延时时间戳较前的失败条目排在列表前面）
final int half = tmpList.size() / 2;
if (half <= 0) { // 长度除2取余小于等于0，说明元素个数为1，直接返回
return tmpList.get(0).getName();
} else { // 元素个数大于1
final int i = this.whichItemWorst.incrementAndGet() % half;
return tmpList.get(i).getName();
}
}
return null;
}
}
以上便是在开启故障延时机制下选择消息队列的源码分析
接着再来了解在关闭故障延时的情况下是如何选择消息发送队列：
public MessageQueue selectOneMessageQueue(final String lastBrokerName) {
if (lastBrokerName == null) { // 上一次发送消息选择的Broker名为空，直接选择下一个消息队列
return selectOneMessageQueue();
} else {
// Broker名不为空，则从上一次发送消息的Broker后一位开始轮询判断，找到不同于上次发送消息的Broker，并返回（实际上就是采用轮询策略选择接受消息的队列）
for (int i = 0; i < this.messageQueueList.size(); i++) {
int index = this.sendWhichQueue.incrementAndGet();
int pos = Math.abs(index) % this.messageQueueList.size();
if (pos < 0)
pos = 0;
MessageQueue mq = this.messageQueueList.get(pos);
if (!mq.getBrokerName().equals(lastBrokerName)) {
return mq;
}
}
return selectOneMessageQueue();
}
}
public MessageQueue selectOneMessageQueue() {
// 轮询获取下一个消息队列
int index = this.sendWhichQueue.incrementAndGet();
int pos = Math.abs(index) % this.messageQueueList.size();
if (pos < 0)
pos = 0;
return this.messageQueueList.get(pos);
}
- 给目标队列发送消息：
private SendResult sendKernelImpl(final Message msg,
final MessageQueue mq,
final CommunicationMode communicationMode,
final SendCallback sendCallback,
final TopicPublishInfo topicPublishInfo,
final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
long beginStartTime = System.currentTimeMillis();
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); // 获取Broker地址
if (null == brokerAddr) { // 找不到则让NameServer更新Broker地址后，再尝试获取
tryToFindTopicPublishInfo(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
}
SendMessageContext context = null;
if (brokerAddr != null) {
brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr);
byte[] prevBody = msg.getBody();
try {
if (!(msg instanceof MessageBatch)) {
MessageClientIDSetter.setUniqID(msg); // 为消息分配唯一id
}
boolean topicWithNamespace = false;
if (null != this.mQClientFactory.getClientConfig().getNamespace()) {
msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace());
topicWithNamespace = true;
}
// 消息大小超过4k，进行压缩处理
int sysFlag = 0;
boolean msgBodyCompressed = false;
if (this.tryToCompressMessage(msg)) {
sysFlag |= MessageSysFlag.COMPRESSED_FLAG;
msgBodyCompressed = true;
}
// 如果消息属于事务类型，则将消息标志为事务消息
final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (tranMsg != null && Boolean.parseBoolean(tranMsg)) {
sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;
}
// 如果有注册到相应的钩子函数，则先执行函数（相当于代码中的增强逻辑）
if (hasCheckForbiddenHook()) {
CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext();
checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr());
checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup());
checkForbiddenContext.setCommunicationMode(communicationMode);
checkForbiddenContext.setBrokerAddr(brokerAddr);
checkForbiddenContext.setMessage(msg);
checkForbiddenContext.setMq(mq);
checkForbiddenContext.setUnitMode(this.isUnitMode());
this.executeCheckForbiddenHook(checkForbiddenContext);
}
if (this.hasSendMessageHook()) {
context = new SendMessageContext();
context.setProducer(this);
context.setProducerGroup(this.defaultMQProducer.getProducerGroup());
context.setCommunicationMode(communicationMode);
context.setBornHost(this.defaultMQProducer.getClientIP());
context.setBrokerAddr(brokerAddr);
context.setMessage(msg);
context.setMq(mq);
context.setNamespace(this.defaultMQProducer.getNamespace());
String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (isTrans != null && isTrans.equals("true")) {
context.setMsgType(MessageType.Trans_Msg_Half);
}
if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) {
context.setMsgType(MessageType.Delay_Msg);
}
this.executeSendMessageHookBefore(context);
}
// 封装消息发送请求头
SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup());
requestHeader.setTopic(msg.getTopic());
requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey());
requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums());
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setSysFlag(sysFlag);
requestHeader.setBornTimestamp(System.currentTimeMillis());
requestHeader.setFlag(msg.getFlag());
requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
requestHeader.setReconsumeTimes(0);
requestHeader.setUnitMode(this.isUnitMode());
requestHeader.setBatch(msg instanceof MessageBatch);
if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
String reconsumeTimes = MessageAccessor.getReconsumeTime(msg);
if (reconsumeTimes != null) {
requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes));
MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME);
}
String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg);
if (maxReconsumeTimes != null) {
requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes));
MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES);
}
}
// 判断请求发送类型
SendResult sendResult = null;
switch (communicationMode) {
case ASYNC: // 异步消息发送
Message tmpMessage = msg;
boolean messageCloned = false;
if (msgBodyCompressed) {
tmpMessage = MessageAccessor.cloneMessage(msg);
messageCloned = true;
msg.setBody(prevBody);
}
if (topicWithNamespace) {
if (!messageCloned) {
tmpMessage = MessageAccessor.cloneMessage(msg);
messageCloned = true;
}
msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
}
long costTimeAsync = System.currentTimeMillis() - beginStartTime;
if (timeout < costTimeAsync) {
throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
tmpMessage,
requestHeader,
timeout - costTimeAsync,
communicationMode,
sendCallback,
topicPublishInfo,
this.mQClientFactory,
this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(),
context,
this);
break;
case ONEWAY: // 单向消息发送
case SYNC: // 同步消息发送
long costTimeSync = System.currentTimeMillis() - beginStartTime;
if (timeout < costTimeSync) {
throw new RemotingTooMuchRequestException("sendKernelImpl call timeout");
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
msg,
requestHeader,
timeout - costTimeSync,
communicationMode,
context,
this);
break;
default:
assert false;
break;
}
// 执行发送消息后的钩子函数
if (this.hasSendMessageHook()) {
context.setSendResult(sendResult);
this.executeSendMessageHookAfter(context);
}
return sendResult; // 返回消息响应结果
} catch (RemotingException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} catch (MQBrokerException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} catch (InterruptedException e) {
if (this.hasSendMessageHook()) {
context.setException(e);
this.executeSendMessageHookAfter(context);
}
throw e;
} finally {
msg.setBody(prevBody);
msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace()));
}
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
到此，Producer的核心源码解析结束
854
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
