消息存储
原理图
- 事务消息包括half消息,提交消息以及回滚消息
- 分别由SendMessageProcessor和EndTransactionProcessor处理
- half消息发送至RMQ_SYS_TRANS_HALF_TOPIC
- 提交消息则发送至业务侧定义topic,并删除half消息
- 回滚消息仅删除half消息,删除的实质是发送消息至RMQ_SYS_TRANS_OP_HALF_TOPIC
- 删除是为了协助消息回查RMQ_SYS_TRANS_HALF_TOPIC避免二次处理
源码分析[half消息]一SendMessageProcessor.sendMessage
- 如果消息sysflag携带事务标记
- 则交由TransactionalMessageService处理
- 调用桥接器进行存储
- 进行消息topic转换后通过store.putMessage存储消息
private RemotingCommand sendMessage(final ChannelHandlerContext ctx,
final RemotingCommand request,
final SendMessageContext sendMessageContext,
final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
...... 删除其他代码
处理事务消息还是普通消息
if (traFlag != null && Boolean.parseBoolean(traFlag)
处理事务消息 内部会将topic转化成事务topic
putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner);
return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt);
}
public PutMessageResult prepareMessage(MessageExtBrokerInner messageInner) {
通过桥接器[桥接DefaultMessageStore]进行存储
return transactionalMessageBridge.putHalfMessage(messageInner);
}
源码分析一parseHalfMessageInner
- 通过Msg Property属性进行topic转换
- 将消息发送至RMQ_SYS_TRANS_HALF_TOPIC
- 将原topic存储至PROPERTY_REAL_TOPIC
private MessageExtBrokerInner parseHalfMessageInner(MessageExtBrokerInner msgInner) {
记住自己真实的topic和队列用于将来转换
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC, msgInner.getTopic());
MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID,
String.valueOf(msgInner.getQueueId()));
此时在设置类型为普通消息 [构建逻辑队列的时候如果是事务类型消息会跳过 这里转一下 用户构建逻辑队列]
msgInner.setSysFlag(
MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), MessageSysFlag.TRANSACTION_NOT_TYPE));
设置到事务消息topic: RMQ_SYS_TRANS_HALF_TOPIC half队列名称
msgInner.setTopic(TransactionalMessageUtil.buildHalfTopic());
half消息只有一个队列 设置成0
msgInner.setQueueId(0);
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
return msgInner;
}
源码分析[commit|rollback]一EndTransactionProcessor.processRequest
- 区分请求为提交消息请求还是回滚消息请求
- commit请求: 查询消息,还原消息转存到原topic
- commit请求: 删除half topic上消息,转储到RMQ_SYS_TRANS_OP_HALF_TOPIC
- rollback请求: 删除half topic上消息,转储到RMQ_SYS_TRANS_OP_HALF_TOPIC
public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws
RemotingCommandException {
...... 删除其他代码
OperationResult result = new OperationResult();
if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) {
查询消息
result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader);
if (result.getResponseCode() == ResponseCode.SUCCESS) {
检查消息
RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);
还原消息
if (res.getCode() == ResponseCode.SUCCESS) {
MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage());
msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));
msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());
msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());
msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp());
MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED);
落盘commit消息 [消费者可消费] 发送到真实主题
RemotingCommand sendResult = sendFinalMessage(msgInner);
删除prepare消息[注意commitlog是appendOnly的操作 这里的删除实现如下]
消息放入RMQ_SYS_TRANS_OP_HALF_TOPIC的主题 , OP_HALF_TOPIC与half_topic一对一
消息内容为half消息 的queueoffset 并设置TAGS的属性为'd'
if (sendResult.getCode() == ResponseCode.SUCCESS) {
this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());
}
return sendResult;
}
return res;
}
} else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {
rollback 逻辑
result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader);
if (result.getResponseCode() == ResponseCode.SUCCESS) {
RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);
if (res.getCode() == ResponseCode.SUCCESS) {
插入op halftopic 但是没有还原真实消息落盘 所以rollback消息永远不会被消费
this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());
}
return res;
}
}
response.setCode(result.getResponseCode());
response.setRemark(result.getResponseRemark());
return response;
}
deletePrepareMessage
- 三部曲:通过桥接器删除消息
- 构建MessageQueue信息并执行topic消息存储
- 写入Op主题RMQ_SYS_TRANS_OP_HALF_TOPIC
public boolean deletePrepareMessage(MessageExt msgExt) {
处理Op消息
if (this.transactionalMessageBridge.putOpMessage(msgExt, TransactionalMessageUtil.REMOVETAG)) {
return true;
}
}
public boolean putOpMessage(MessageExt messageExt, String opType) {
MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(),
this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId());
if (TransactionalMessageUtil.REMOVETAG.equals(opType)) {
添加到Op主题RMQ_SYS_TRANS_OP_HALF_TOPIC
return addRemoveTagInTransactionOp(messageExt, messageQueue);
}
return true;
}
private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) {
构建RMQ_SYS_TRANS_OP_HALF_TOPIC主题
Message message = new Message(TransactionalMessageUtil.buildOpTopic(), TransactionalMessageUtil.REMOVETAG,
String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionalMessageUtil.charset));
写入Op消息到messageQueue [底层调用store.putMessage]
writeOp(message, messageQueue);
return true;
}
消息回查
- 回查线程每一分钟执行一次
- 处理超过6秒未回查消息
- 超过15次则丢弃消息
public class TransactionalMessageCheckService extends ServiceThread {
@Override
public void run() {
while (!this.isStopped()) {
1分钟等待
this.waitForRunning(checkInterval);
}
}
等待结束则执行回查
@Override
protected void onWaitEnd() {
超过6秒需要回查
long timeout = brokerController.getBrokerConfig().getTransactionTimeOut();
int checkMax = brokerController.getBrokerConfig().getTransactionCheckMax();
long begin = System.currentTimeMillis();
事务消息回查
this.brokerController.getTransactionalMessageService().check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener());
}
}
源码分析一check
- 根据transactionTimeout,transactionCheckMax以及用户自定义checkImmunityTimeStr来决定是否回查消息
- 回查消息通过异步调用请求触发生产者TransactionListener.checkLocalTransaction
- 生产者则再次发送相关请求至EndTransactionProcessor
- 超过15次回查则丢弃消息至TRANS_CHECK_MAX_TIME_TOPIC
- 通过RMQ_SYS_TRANS_OP_HALF_TOPIC判断消息是否已经被删除,删除则无需处理
- 一次仅拉取32条RMQ_SYS_TRANS_OP_HALF_TOPIC,如果存在未成功匹配,则多次拉取
public void check(long transactionTimeout, int transactionCheckMax,
AbstractTransactionalMessageCheckListener listener) {
try {
String topic = MixAll.RMQ_SYS_TRANS_HALF_TOPIC;
Set<MessageQueue> msgQueues = transactionalMessageBridge.fetchMessageQueues(topic);
if (msgQueues == null || msgQueues.size() == 0) {
log.warn("The queue of topic is empty :" + topic);
return;
}
log.debug("Check topic={}, queues={}", topic, msgQueues);
for (MessageQueue messageQueue : msgQueues) {
long startTime = System.currentTimeMillis();
MessageQueue opQueue = getOpQueue(messageQueue);
long halfOffset = transactionalMessageBridge.fetchConsumeOffset(messageQueue);
long opOffset = transactionalMessageBridge.fetchConsumeOffset(opQueue);
log.info("Before check, the queue={} msgOffset={} opOffset={}", messageQueue, halfOffset, opOffset);
if (halfOffset < 0 || opOffset < 0) {
log.error("MessageQueue: {} illegal offset read: {}, op offset: {},skip this queue", messageQueue,
halfOffset, opOffset);
continue;
}
List<Long> doneOpOffset = new ArrayList<>();
HashMap<Long, Long> removeMap = new HashMap<>();
PullResult pullResult = fillOpRemoveMap(removeMap, opQueue, opOffset, halfOffset, doneOpOffset);
if (null == pullResult) {
log.error("The queue={} check msgOffset={} with opOffset={} failed, pullResult is null",
messageQueue, halfOffset, opOffset);
continue;
}
int getMessageNullCount = 1;
long newOffset = halfOffset;
long i = halfOffset;
while (true) {
if (System.currentTimeMillis() - startTime > MAX_PROCESS_TIME_LIMIT) {
log.info("Queue={} process time reach max={}", messageQueue, MAX_PROCESS_TIME_LIMIT);
break;
}
if (removeMap.containsKey(i)) {
log.info("Half offset {} has been committed/rolled back", i);
Long removedOpOffset = removeMap.remove(i);
doneOpOffset.add(removedOpOffset);
} else {
GetResult getResult = getHalfMsg(messageQueue, i);
MessageExt msgExt = getResult.getMsg();
if (msgExt == null) {
if (getMessageNullCount++ > MAX_RETRY_COUNT_WHEN_HALF_NULL) {
break;
}
if (getResult.getPullResult().getPullStatus() == PullStatus.NO_NEW_MSG) {
log.debug("No new msg, the miss offset={} in={}, continue check={}, pull result={}", i,
messageQueue, getMessageNullCount, getResult.getPullResult());
break;
} else {
log.info("Illegal offset, the miss offset={} in={}, continue check={}, pull result={}",
i, messageQueue, getMessageNullCount, getResult.getPullResult());
i = getResult.getPullResult().getNextBeginOffset();
newOffset = i;
continue;
}
}
if (needDiscard(msgExt, transactionCheckMax) || needSkip(msgExt)) {
listener.resolveDiscardMsg(msgExt);
newOffset = i + 1;
i++;
continue;
}
if (msgExt.getStoreTimestamp() >= startTime) {
log.debug("Fresh stored. the miss offset={}, check it later, store={}", i,
new Date(msgExt.getStoreTimestamp()));
break;
}
long valueOfCurrentMinusBorn = System.currentTimeMillis() - msgExt.getBornTimestamp();
long checkImmunityTime = transactionTimeout;
String checkImmunityTimeStr = msgExt.getUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS);
if (null != checkImmunityTimeStr) {
checkImmunityTime = getImmunityTime(checkImmunityTimeStr, transactionTimeout);
if (valueOfCurrentMinusBorn < checkImmunityTime) {
if (checkPrepareQueueOffset(removeMap, doneOpOffset, msgExt)) {
newOffset = i + 1;
i++;
continue;
}
}
} else {
if ((0 <= valueOfCurrentMinusBorn) && (valueOfCurrentMinusBorn < checkImmunityTime)) {
log.debug("New arrived, the miss offset={}, check it later checkImmunity={}, born={}", i,
checkImmunityTime, new Date(msgExt.getBornTimestamp()));
break;
}
}
List<MessageExt> opMsg = pullResult.getMsgFoundList();
boolean isNeedCheck = (opMsg == null && valueOfCurrentMinusBorn > checkImmunityTime)
|| (opMsg != null && (opMsg.get(opMsg.size() - 1).getBornTimestamp() - startTime > transactionTimeout))
|| (valueOfCurrentMinusBorn <= -1);
if (isNeedCheck) {
if (!putBackHalfMsgQueue(msgExt, i)) {
continue;
}
listener.resolveHalfMsg(msgExt);
} else {
pullResult = fillOpRemoveMap(removeMap, opQueue, pullResult.getNextBeginOffset(), halfOffset, doneOpOffset);
log.debug("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i,
messageQueue, pullResult);
continue;
}
}
newOffset = i + 1;
i++;
}
if (newOffset != halfOffset) {
transactionalMessageBridge.updateConsumeOffset(messageQueue, newOffset);
}
long newOpOffset = calculateOpOffset(doneOpOffset, opOffset);
if (newOpOffset != opOffset) {
transactionalMessageBridge.updateConsumeOffset(opQueue, newOpOffset);
}
}
} catch (Throwable e) {
log.error("Check error", e);
}
}
总结
- 事务消息通过生产者主动发送以及broker回查共同完成
- 回查最大次数为15次
- 回查需要RMQ_SYS_TRANS_OP_HALF_TOPIC和RMQ_SYS_TRANS_HALF_TOPIC协助工作
- 消息最开始发送至half队列,等待commit提交完成后发送至业务队列供消费者消费