源码分析RocketMQ之CommitLog消息存储机制

long beginTime = this.getSystemClock().now();

PutMessageResult result = this.commitLog.putMessage(msg); // @2

long eclipseTime = this.getSystemClock().now() - beginTime;

if (eclipseTime > 500) {

log.warn(“putMessage not in lock eclipse time(ms)={}, bodyLength={}”, eclipseTime, msg.getBody().length);

}

this.storeStatsService.setPutMessageEntireTimeMax(eclipseTime); //@3

if (null == result || !result.isOk()) { //@4

this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();

}

return result;

}

代码@1：检测操作系统页写入是否繁忙。

@Override

public boolean isOSPageCacheBusy() {

long begin = this.getCommitLog().getBeginTimeInLock();

long diff = this.systemClock.now() - begin;

if (diff < 10000000 //

&& diff > this.messageStoreConfig.getOsPageCacheBusyTimeOutMills()) {

return true;

}

return false;

}

代码@2：将日志写入CommitLog 文件，具体实现类 CommitLog。

代码@3：记录相关统计信息。

代码@4：记录写commitlog 失败次数。

1.2.2 CommitLog.putMessage

public PutMessageResult putMessage(final MessageExtBrokerInner msg) {

// Set the storage time

msg.setStoreTimestamp(System.currentTimeMillis());

// Set the message body BODY CRC (consider the most appropriate setting

// on the client)

msg.setBodyCRC(UtilAll.crc32(msg.getBody()));

// Back to Results

AppendMessageResult result = null;

StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();

String topic = msg.getTopic();

int queueId = msg.getQueueId();

final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); // @1

if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE//

|| tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { // @2

// Delay Delivery

if (msg.getDelayTimeLevel() > 0) {

if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) {

msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel());

}

topic = ScheduleMessageService.SCHEDULE_TOPIC;

queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel());

// Backup real topic, queueId

MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());

MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));

msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));

msg.setTopic(topic);

msg.setQueueId(queueId);

}

}

long eclipseTimeInLock = 0;

MappedFile unlockMappedFile = null;

MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); // @3

putMessageLock.lock(); //spin or ReentrantLock ,depending on store config //@4

try {

long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();

this.beginTimeInLock = beginLockTimestamp;

// Here settings are stored timestamp, in order to ensure an orderly

// global

msg.setStoreTimestamp(beginLockTimestamp);

if (null == mappedFile || mappedFile.isFull()) {

mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise

}

if (null == mappedFile) {

log.error("create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());

beginTimeInLock = 0;

return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null);

} // @5

result = mappedFile.appendMessage(msg, this.appendMessageCallback); // @6

switch (result.getStatus()) {

case PUT_OK:

break;

case END_OF_FILE:

unlockMappedFile = mappedFile;

// Create a new file, re-write the message

mappedFile = this.mappedFileQueue.getLastMappedFile(0);

if (null == mappedFile) {

// XXX: warn and notify me

log.error("create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());

beginTimeInLock = 0;

return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result);

}

result = mappedFile.appendMessage(msg, this.appendMessageCallback);

break;

case MESSAGE_SIZE_EXCEEDED:

case PROPERTIES_SIZE_EXCEEDED:

beginTimeInLock = 0;

return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result);

case UNKNOWN_ERROR:

beginTimeInLock = 0;

return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);

default:

beginTimeInLock = 0;

return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);

}

eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;

beginTimeInLock = 0;

} finally {

putMessageLock.unlock();

}

if (eclipseTimeInLock > 500) {

log.warn(“[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}”, eclipseTimeInLock, msg.getBody().length, result);

}

if (null != unlockMappedFile && this.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {

this.defaultMessageStore.unlockMappedFile(unlockMappedFile);

}

PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);

// Statistics

storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();

storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes());

handleDiskFlush(result, putMessageResult, msg); // @7

handleHA(result, putMessageResult, msg); //@8

return putMessageResult;

}

先对 ComitLog 写入消息做一个简单描述，然后需要详细探究每个步骤的实现。

代码@1：获取消息类型（事务消息，非事务消息，Commit消息。

代码@3：获取一个 MappedFile 对象，内存映射的具体实现。

代码@4，追加消息需要加锁，串行化处理。

代码@5：验证代码@3的 MappedFile 对象，获取一个可用的 MappedFile (如果没有，则创建一个)。

代码@6：通过MappedFile对象写入文件。

代码@7：根据刷盘策略刷盘。

代码@8：主从同步。

1.3 存储核心类分析

---

1.3.1 源码分析MappedFile

1.3.1.1 MappedFile 基础属性

public static final int OS_PAGE_SIZE = 1024 * 4; // 4K

private static final AtomicLong TOTAL_MAPPED_VIRTUAL_MEMORY = new AtomicLong(0);

private static final AtomicInteger TOTAL_MAPPED_FILES = new AtomicInteger(0);

protected final AtomicInteger wrotePosition = new AtomicInteger(0);

protected final AtomicInteger committedPosition = new AtomicInteger(0);

private final AtomicInteger flushedPosition = new AtomicInteger(0);

protected int fileSize;

protected FileChannel fileChannel;

/**

• Message will put to here first, and then reput to FileChannel if writeBuffer is not null.

*/

protected ByteBuffer writeBuffer = null;

protected TransientStorePool transientStorePool = null;

private String fileName;

private long fileFromOffset;

private File file;

private MappedByteBuffer mappedByteBuffer;

private volatile long storeTimestamp = 0;

private boolean firstCreateInQueue = false;

• OS_PAGE_SIZE

OSpage大小，4K。

• TOTAL_MAPPED_VIRTUAL_MEMORY

类变量，所有 MappedFile 实例已使用字节总数。

• TOTAL_MAPPED_FILES

MappedFile 个数。

• wrotePosition

当前MappedFile对象当前写指针。

• committedPosition

当前提交的指针。

• flushedPosition

当前刷写到磁盘的指针。

• fileSize

文件总大小。

• fileChannel

文件通道。

• writeBuffer

如果开启了transientStorePoolEnable，消息会写入堆外内存，然后提交到 PageCache 并最终刷写到磁盘。

• TransientStorePool transientStorePool

ByteBuffer的缓冲池，堆外内存，transientStorePoolEnable 为 true 时生效。

• fileName

文件名称。

• fileFromOffset

文件序号,代表该文件代表的文件偏移量。

• File file

文件对象。

• MappedByteBuffer mappedByteBuffer

对应操作系统的 PageCache。

• storeTimestamp

最后一次存储时间戳。

1.3.1.2 初始化

private void init(final String fileName, final int fileSize) throws IOException {

this.fileName = fileName;

this.fileSize = fileSize;

this.file = new File(fileName);

this.fileFromOffset = Long.parseLong(this.file.getName());

boolean ok = false;

ensureDirOK(this.file.getParent());

try {

this.fileChannel = new RandomAccessFile(this.file, “rw”).getChannel();

this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize);

TOTAL_MAPPED_VIRTUAL_MEMORY.addAndGet(fileSize);

TOTAL_MAPPED_FILES.incrementAndGet();

ok = true;

} catch (FileNotFoundException e) {

log.error("create file channel " + this.fileName + " Failed. ", e);

throw e;

} catch (IOException e) {

log.error("map file " + this.fileName + " Failed. ", e);

throw e;

} finally {

if (!ok && this.fileChannel != null) {

this.fileChannel.close();

}

}

}

初始化 FileChannel、mappedByteBuffer 等。

1.3.1.3 appendMessagesInner 消息写入

public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb) {

assert messageExt != null;

assert cb != null;

int currentPos = this.wrotePosition.get(); // @1

if (currentPos < this.fileSize) {

ByteBuffer byteBuffer = writeBuffer != null ? writeBuffer.slice() : this.mappedByteBuffer.slice();

byteBuffer.position(currentPos);

AppendMessageResult result = null;

if (messageExt instanceof MessageExtBrokerInner) { // @2

result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBrokerInner) messageExt);

} else if (messageExt instanceof MessageExtBatch) {

result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBatch)messageExt);

} else {

return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);

}

this.wrotePosition.addAndGet(result.getWroteBytes()); // @4

this.storeTimestamp = result.getStoreTimestamp();

return result;

}

log.error(“MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}”, currentPos, this.fileSize);

return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);

}

代码@1：获取当前写入位置。

代码@2：根据消息类型，是批量消息还是单个消息，进入相应的处理。

代码@3：消息写入实现。

接下看具体的消息写入逻辑，代码来源于 CommitLog$DefaultAppendMessageCallback。

public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank,

final MessageExtBrokerInner msgInner) { //@1

// STORETIMESTAMP + STOREHOSTADDRESS + OFFSET

// PHY OFFSET

long wroteOffset = fileFromOffset + byteBuffer.position();

this.resetByteBuffer(hostHolder, 8);

String msgId = MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(hostHolder), wroteOffset); //@2

// Record ConsumeQueue information //@3start

keyBuilder.setLength(0);

keyBuilder.append(msgInner.getTopic());

keyBuilder.append(‘-’);

keyBuilder.append(msgInner.getQueueId());

String key = keyBuilder.toString();

Long queueOffset = CommitLog.this.topicQueueTable.get(key);

if (null == queueOffset) {

queueOffset = 0L;

CommitLog.this.topicQueueTable.put(key, queueOffset);

} //@3 end

// Transaction messages that require special handling //@4 start

final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag());

switch (tranType) {

// Prepared and Rollback message is not consumed, will not enter the

// consumer queuec

case MessageSysFlag.TRANSACTION_PREPARED_TYPE:

case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:

queueOffset = 0L;

break;

case MessageSysFlag.TRANSACTION_NOT_TYPE:

case MessageSysFlag.TRANSACTION_COMMIT_TYPE:

default:

break;

} // @4 end

/**

• Serialize message

*/

final byte[] propertiesData =

msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);

final int propertiesLength = propertiesData == null ? 0 : propertiesData.length;

if (propertiesLength > Short.MAX_VALUE) {

log.warn(“putMessage message properties length too long. length={}”, propertiesData.length);

return new AppendMessageResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED);

} //@5

final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);

final int topicLength = topicData.length;

final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;

final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength); //@6

// Exceeds the maximum message

if (msgLen > this.maxMessageSize) { // @7

CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength

• ", maxMessageSize: " + this.maxMessageSize);

return new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED);

}

// Determines whether there is sufficient free space

if ((msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) { // @8

this.resetByteBuffer(this.msgStoreItemMemory, maxBlank);

// 1 TOTALSIZE

this.msgStoreItemMemory.putInt(maxBlank);

// 2 MAGICCODE

this.msgStoreItemMemory.putInt(CommitLog.BLANK_MAGIC_CODE);

// 3 The remaining space may be any value

//

// Here the length of the specially set maxBlank

final long beginTimeMills = CommitLog.this.defaultMessageStore.now();

byteBuffer.put(this.msgStoreItemMemory.array(), 0, maxBlank);

return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgId, msgInner.getStoreTimestamp(),

queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);

}

// Initialization of storage space @9

this.resetByteBuffer(msgStoreItemMemory, msgLen);

// 1 TOTALSIZE

this.msgStoreItemMemory.putInt(msgLen);

// 2 MAGICCODE

this.msgStoreItemMemory.putInt(CommitLog.MESSAGE_MAGIC_CODE);

// 3 BODYCRC

this.msgStoreItemMemory.putInt(msgInner.getBodyCRC());

// 4 QUEUEID

this.msgStoreItemMemory.putInt(msgInner.getQueueId());

// 5 FLAG

this.msgStoreItemMemory.putInt(msgInner.getFlag());

// 6 QUEUEOFFSET

this.msgStoreItemMemory.putLong(queueOffset);

// 7 PHYSICALOFFSET

this.msgStoreItemMemory.putLong(fileFromOffset + byteBuffer.position());

// 8 SYSFLAG

this.msgStoreItemMemory.putInt(msgInner.getSysFlag());

// 9 BORNTIMESTAMP

this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp());

// 10 BORNHOST

this.resetByteBuffer(hostHolder, 8);

this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder));

// 11 STORETIMESTAMP

this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp());

// 12 STOREHOSTADDRESS

this.resetByteBuffer(hostHolder, 8);

this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder));

//this.msgBatchMemory.put(msgInner.getStoreHostBytes());

// 13 RECONSUMETIMES

this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes());

// 14 Prepared Transaction Offset

this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset());

// 15 BODY

this.msgStoreItemMemory.putInt(bodyLength);

if (bodyLength > 0)

this.msgStoreItemMemory.put(msgInner.getBody());

// 16 TOPIC

this.msgStoreItemMemory.put((byte) topicLength);

this.msgStoreItemMemory.put(topicData);

// 17 PROPERTIES

this.msgStoreItemMemory.putShort((short) propertiesLength);

if (propertiesLength > 0)

this.msgStoreItemMemory.put(propertiesData);

final long beginTimeMills = CommitLog.this.defaultMessageStore.now();

// Write messages to the queue buffer

byteBuffer.put(this.msgStoreItemMemory.array(), 0, msgLen);

AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgId,

msgInner.getStoreTimestamp(), queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); //@10

switch (tranType) {

case MessageSysFlag.TRANSACTION_PREPARED_TYPE:

case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:

break;

case MessageSysFlag.TRANSACTION_NOT_TYPE:

case MessageSysFlag.TRANSACTION_COMMIT_TYPE:

// The next update ConsumeQueue information

CommitLog.this.topicQueueTable.put(key, ++queueOffset);

break;

default:

break;

}

return result;

}

代码@1：参数详解

• fileFromOffset

该文件在整个文件序列中的偏移量。

• ByteBuffer byteBuffer

byteBuffer，NIO 字节容器。

• int maxBlank

最大可写字节数。

• MessageExtBrokerInner msgInner

消息内部封装实体。

代码@2：创建msgId，底层存储由16个字节表示，如下图：

代码@3：根据 topic-queryId 获取该队列的偏移地址（待写入的地址），如果没有，新增一个键值对，当前偏移量为 0。

代码@4：对事务消息需要单独特殊的处理(PREPARE,ROLLBACK类型的消息，不进入Consume队列)。

代码@5：消息的附加属性长度不能超过65536个字节。

代码@6：计算消息存储长度，消息存储格式：

代码@7：如果消息长度超过配置的消息总长度，则返回 MESSAGE_SIZE_EXCEEDED。

代码@8：如果该 MapperFile 中可剩余空间小于当前消息存储空间，返回END_OF_FILE。

代码@9：将消息写入MapperFile中（内存中）。

代码@10：重点讲解一下AppendMessageResult方法。

public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, String msgId,

long storeTimestamp, long logicsOffset, long pagecacheRT) {

this.status = status;

this.wroteOffset = wroteOffset;

this.wroteBytes = wroteBytes;

this.msgId = msgId;

this.storeTimestamp = storeTimestamp;

this.logicsOffset = logicsOffset;

this.pagecacheRT = pagecacheRT;

}

• AppendMessageStatus status

追加结果（成功，到达文件尾（文件剩余空间不足）、消息长度超过、消息属性长度超出、未知错误）。

• wroteOffset

消息的偏移量（相对于整个commitlog）。

• wroteBytes

消息待写入字节。

• msgId

消息ID。

• storeTimestamp

消息写入时间戳。

• logicsOffset

消息队列偏移量。

• pagecacheRT

消息写入时机戳（消息存储时间戳— 消息存储开始时间戳）。

然后返回 AppendMessageStatus，流程回到 【1.2.2 CommitLog.putMessage】的代码@6，如果返回结果是 OK 的话进入到代码@7，@8。

handleDiskFlush(result, putMessageResult, msg);   // @7

handleHA(result, putMessageResult, msg);            //@8

我们先关注正常流程，然后再次梳理流程，特别关注异常流程，锁竞争等情况。

2、 消息刷盘

===========

public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {

// Synchronization flush

if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { // @1

final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;

if (messageExt.isWaitStoreMsgOK()) { // @2

GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());

service.putRequest(request);

boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());

if (!flushOK) {

log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()

• " client address: " + messageExt.getBornHostString());

putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);

}

} else { //@3

service.wakeup();

}

}

// Asynchronous flush //@4

else {

if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {

flushCommitLogService.wakeup();

} else {

commitLogService.wakeup();

}

}

}

刷写磁盘支持同步、异步刷写，下文将分别重点阐述该这两种机制。

代码@1：同步刷写，这里有两种配置，是否一定要收到存储MSG信息，才返回，默认为true。

代码@2：如果要等待存储结果。

代码@3：唤醒同步刷盘线程。

代码@4：异步刷盘机制。

2.1 GroupCommitRequest (同步刷盘)

---

先来分析一下 GroupCommitRequest 。

面试资料整理汇总

这些面试题是我朋友进阿里前狂刷七遍以上的面试资料，由于面试文档很多，内容更多，没有办法一一为大家展示出来，所以只好为大家节选出来了一部分供大家参考。

面试的本质不是考试，而是告诉面试官你会做什么，所以，这些面试资料中提到的技术也是要学会的，不然稍微改动一下你就凉凉了

在这里祝大家能够拿到心仪的offer！
t.getTags()

• " client address: " + messageExt.getBornHostString());

putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);

}

} else { //@3

service.wakeup();

}

}

// Asynchronous flush //@4

else {

if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {

flushCommitLogService.wakeup();

} else {

commitLogService.wakeup();

}

}

}

刷写磁盘支持同步、异步刷写，下文将分别重点阐述该这两种机制。

代码@1：同步刷写，这里有两种配置，是否一定要收到存储MSG信息，才返回，默认为true。

代码@2：如果要等待存储结果。

代码@3：唤醒同步刷盘线程。

代码@4：异步刷盘机制。

2.1 GroupCommitRequest (同步刷盘)

---

先来分析一下 GroupCommitRequest 。

面试资料整理汇总

[外链图片转存中…(img-qU5C7c6F-1720118797843)]

[外链图片转存中…(img-xILuTn3S-1720118797844)]

这些面试题是我朋友进阿里前狂刷七遍以上的面试资料，由于面试文档很多，内容更多，没有办法一一为大家展示出来，所以只好为大家节选出来了一部分供大家参考。

面试的本质不是考试，而是告诉面试官你会做什么，所以，这些面试资料中提到的技术也是要学会的，不然稍微改动一下你就凉凉了

在这里祝大家能够拿到心仪的offer！