rocketmq源码解析结束事务处理器①

最新推荐文章于 2023-01-13 16:56:43 发布

chucuavins01156

最新推荐文章于 2023-01-13 16:56:43 发布

阅读量144

点赞数

文章标签： java 开发工具

原文链接：https://my.oschina.net/u/3775437/blog/3097014

版权

说在前面

结束事务处理器

源码解析

进入这个方法，org.apache.rocketmq.broker.processor.EndTransactionProcessor#processRequest

 @Override    public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws        RemotingCommandException {        final RemotingCommand response = RemotingCommand.createResponseCommand(null);        final EndTransactionRequestHeader requestHeader =            (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);        LOGGER.info("Transaction request:{}", requestHeader);//        如果broker是从节点        if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {            response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE);            LOGGER.warn("Message store is slave mode, so end transaction is forbidden. ");            return response;        }
        if (requestHeader.getFromTransactionCheck()) {            switch (requestHeader.getCommitOrRollback()) {                case MessageSysFlag.TRANSACTION_NOT_TYPE: {                    LOGGER.warn("Check producer[{}] transaction state, but it's pending status."                            + "RequestHeader: {} Remark: {}",                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),                        requestHeader.toString(),                        request.getRemark());                    return null;                }
                case MessageSysFlag.TRANSACTION_COMMIT_TYPE: {                    LOGGER.warn("Check producer[{}] transaction state, the producer commit the message."                            + "RequestHeader: {} Remark: {}",                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),                        requestHeader.toString(),                        request.getRemark());
                    break;                }
                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {                    LOGGER.warn("Check producer[{}] transaction state, the producer rollback the message."                            + "RequestHeader: {} Remark: {}",                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),                        requestHeader.toString(),                        request.getRemark());                    break;                }                default:                    return null;            }        } else {            switch (requestHeader.getCommitOrRollback()) {                case MessageSysFlag.TRANSACTION_NOT_TYPE: {                    LOGGER.warn("The producer[{}] end transaction in sending message,  and it's pending status."                            + "RequestHeader: {} Remark: {}",                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),                        requestHeader.toString(),                        request.getRemark());                    return null;                }
                case MessageSysFlag.TRANSACTION_COMMIT_TYPE: {                    break;                }
                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: {                    LOGGER.warn("The producer[{}] end transaction in sending message, rollback the message."                            + "RequestHeader: {} Remark: {}",                        RemotingHelper.parseChannelRemoteAddr(ctx.channel()),                        requestHeader.toString(),                        request.getRemark());                    break;                }                default:                    return null;            }        }        OperationResult result = new OperationResult();//        事务提交        if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == requestHeader.getCommitOrRollback()) {//            事务消息提交=》            result = this.brokerController.getTransactionalMessageService().commitMessage(requestHeader);            if (result.getResponseCode() == ResponseCode.SUCCESS) {//                =》                RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);                if (res.getCode() == ResponseCode.SUCCESS) {//                    组装结束事务消息=》                    MessageExtBrokerInner msgInner = endMessageTransaction(result.getPrepareMessage());                    msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));                    msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());                    msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());                    msgInner.setStoreTimestamp(result.getPrepareMessage().getStoreTimestamp());//                    发送最终消息=》                    RemotingCommand sendResult = sendFinalMessage(msgInner);                    if (sendResult.getCode() == ResponseCode.SUCCESS) {//                        删除准备消息=》                        this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());                    }                    return sendResult;                }                return res;            }        } else if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {//            事务消息回滚=》            result = this.brokerController.getTransactionalMessageService().rollbackMessage(requestHeader);            if (result.getResponseCode() == ResponseCode.SUCCESS) {                RemotingCommand res = checkPrepareMessage(result.getPrepareMessage(), requestHeader);                if (res.getCode() == ResponseCode.SUCCESS) {//                    删除准备消息=》                    this.brokerController.getTransactionalMessageService().deletePrepareMessage(result.getPrepareMessage());                }                return res;            }        }        response.setCode(result.getResponseCode());        response.setRemark(result.getResponseRemark());        return response;    }

进入这个方法，事务消息提交，org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl#commitMessage

 @Override    public OperationResult commitMessage(EndTransactionRequestHeader requestHeader) {        return getHalfMessageByOffset(requestHeader.getCommitLogOffset());    }

进入这个方法，org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl#getHalfMessageByOffset

 private OperationResult getHalfMessageByOffset(long commitLogOffset) {        OperationResult response = new OperationResult();//        根据offset查询消息=》        MessageExt messageExt = this.transactionalMessageBridge.lookMessageByOffset(commitLogOffset);        if (messageExt != null) {            response.setPrepareMessage(messageExt);            response.setResponseCode(ResponseCode.SUCCESS);        } else {            response.setResponseCode(ResponseCode.SYSTEM_ERROR);            response.setResponseRemark("Find prepared transaction message failed");        }        return response;    }

进入这个方法，根据offset查询消息，org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge#lookMessageByOffset

 public MessageExt lookMessageByOffset(final long commitLogOffset) {        return this.store.lookMessageByOffset(commitLogOffset);    }

进入这个方法，org.apache.rocketmq.store.DefaultMessageStore#lookMessageByOffset(long)

   public MessageExt lookMessageByOffset(long commitLogOffset) {//        根据commitLog查询SelectMappedBufferResult=》        SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4);        if (null != sbr) {            try {                // 1 TOTALSIZE                int size = sbr.getByteBuffer().getInt();//                根据offset和大小查询消息=》                return lookMessageByOffset(commitLogOffset, size);            } finally {                sbr.release();            }        }
        return null;    }

进入这个方法，根据commitLog查询SelectMappedBufferResult，org.apache.rocketmq.store.CommitLog#getMessage

  public SelectMappedBufferResult getMessage(final long offset, final int size) {        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog();//        根据offset找到映射文件 =》        MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, offset == 0);        if (mappedFile != null) {            int pos = (int) (offset % mappedFileSize);            return mappedFile.selectMappedBuffer(pos, size);        }        return null;    }

进入这个方法，根据offset找到映射文件，org.apache.rocketmq.store.MappedFileQueue#findMappedFileByOffset(long, boolean)

 public MappedFile findMappedFileByOffset(final long offset, final boolean returnFirstOnNotFound) {        try {//            获取队列中第一个映射文件            MappedFile firstMappedFile = this.getFirstMappedFile();//            获取队列中最后一个映射文件            MappedFile lastMappedFile = this.getLastMappedFile();            if (firstMappedFile != null && lastMappedFile != null) {//                如果offset不在索引文件的offset范围内                if (offset < firstMappedFile.getFileFromOffset() || offset >= lastMappedFile.getFileFromOffset() + this.mappedFileSize) {                    LOG_ERROR.warn("Offset not matched. Request offset: {}, firstOffset: {}, lastOffset: {}, mappedFileSize: {}, mappedFiles count: {}",                        offset,                        firstMappedFile.getFileFromOffset(),                        lastMappedFile.getFileFromOffset() + this.mappedFileSize,                        this.mappedFileSize,                        this.mappedFiles.size());                } else {//                   找到映射文件在队列中的索引位置                    int index = (int) ((offset / this.mappedFileSize) - (firstMappedFile.getFileFromOffset() / this.mappedFileSize));                    MappedFile targetFile = null;                    try {//                        获取索引文件                        targetFile = this.mappedFiles.get(index);                    } catch (Exception ignored) {                    }
//                    offset在目标文件的起始offset和结束offset范围内                    if (targetFile != null && offset >= targetFile.getFileFromOffset()                        && offset < targetFile.getFileFromOffset() + this.mappedFileSize) {                        return targetFile;                    }
//                    如果按索引在队列中找不到映射文件就遍历队列查找映射文件                    for (MappedFile tmpMappedFile : this.mappedFiles) {                        if (offset >= tmpMappedFile.getFileFromOffset()                            && offset < tmpMappedFile.getFileFromOffset() + this.mappedFileSize) {                            return tmpMappedFile;                        }                    }                }
//                如果offset=0获取队列中第一个映射文件，个人感觉这个逻辑是否放在前面判断更为合理，还是放在这里另有深意                if (returnFirstOnNotFound) {                    return firstMappedFile;                }            }        } catch (Exception e) {            log.error("findMappedFileByOffset Exception", e);        }
        return null;    }

往上返回到这个方法，根据offset和大小查询消息，org.apache.rocketmq.store.DefaultMessageStore#lookMessageByOffset(long, int)

public MessageExt lookMessageByOffset(long commitLogOffset, int size) {//        根据offset和大小查询offset=》        SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, size);        if (null != sbr) {            try {                return MessageDecoder.decode(sbr.getByteBuffer(), true, false);            } finally {                sbr.release();            }        }
        return null;    }

进入这个方法，根据offset和大小查询offset，org.apache.rocketmq.store.CommitLog#getMessage上面介绍过了。

往上返回到这个方法，检查准备提交的事务消息，org.apache.rocketmq.broker.processor.EndTransactionProcessor#checkPrepareMessage

 private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) {        final RemotingCommand response = RemotingCommand.createResponseCommand(null);        if (msgExt != null) {            final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);            if (!pgroupRead.equals(requestHeader.getProducerGroup())) {                response.setCode(ResponseCode.SYSTEM_ERROR);                response.setRemark("The producer group wrong");                return response;            }
            if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) {                response.setCode(ResponseCode.SYSTEM_ERROR);                response.setRemark("The transaction state table offset wrong");                return response;            }
            if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) {                response.setCode(ResponseCode.SYSTEM_ERROR);                response.setRemark("The commit log offset wrong");                return response;            }        } else {            response.setCode(ResponseCode.SYSTEM_ERROR);            response.setRemark("Find prepared transaction message failed");            return response;        }        response.setCode(ResponseCode.SUCCESS);        return response;    }

进入这个方法，发送最终消息，org.apache.rocketmq.broker.processor.EndTransactionProcessor#sendFinalMessage

final RemotingCommand response = RemotingCommand.createResponseCommand(null);//        存储消息=》        final PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);        if (putMessageResult != null) {            switch (putMessageResult.getPutMessageStatus()) {                // Success                case PUT_OK:                case FLUSH_DISK_TIMEOUT:                case FLUSH_SLAVE_TIMEOUT:                case SLAVE_NOT_AVAILABLE:                    response.setCode(ResponseCode.SUCCESS);                    response.setRemark(null);                    break;                // Failed                case CREATE_MAPEDFILE_FAILED:                    response.setCode(ResponseCode.SYSTEM_ERROR);                    response.setRemark("Create mapped file failed.");                    break;//                    消息太大                case MESSAGE_ILLEGAL://                    消息属性太大                case PROPERTIES_SIZE_EXCEEDED:                    response.setCode(ResponseCode.MESSAGE_ILLEGAL);                    response.setRemark("The message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k.");                    break;                case SERVICE_NOT_AVAILABLE:                    response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);                    response.setRemark("Service not available now.");                    break;//                    系统繁忙                case OS_PAGECACHE_BUSY:                    response.setCode(ResponseCode.SYSTEM_ERROR);                    response.setRemark("OS page cache busy, please try another machine");                    break;                case UNKNOWN_ERROR:                    response.setCode(ResponseCode.SYSTEM_ERROR);                    response.setRemark("UNKNOWN_ERROR");                    break;                default:                    response.setCode(ResponseCode.SYSTEM_ERROR);                    response.setRemark("UNKNOWN_ERROR DEFAULT");                    break;            }            return response;        } else {            response.setCode(ResponseCode.SYSTEM_ERROR);            response.setRemark("store putMessage return null");        }        return response;    }

进入这个方法，存储消息，org.apache.rocketmq.store.DefaultMessageStore#putMessage

 public PutMessageResult putMessage(MessageExtBrokerInner msg) {//        存储服务不可用        if (this.shutdown) {            log.warn("message store has shutdown, so putMessage is forbidden");            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);        }
//        broker角色不是master        if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {            long value = this.printTimes.getAndIncrement();            if ((value % 50000) == 0) {                log.warn("message store is slave mode, so putMessage is forbidden ");            }
            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);        }
//        没有写权限        if (!this.runningFlags.isWriteable()) {            long value = this.printTimes.getAndIncrement();            if ((value % 50000) == 0) {                log.warn("message store is not writeable, so putMessage is forbidden " + this.runningFlags.getFlagBits());            }
            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);        } else {            this.printTimes.set(0);        }
//        topic长度不合法        if (msg.getTopic().length() > Byte.MAX_VALUE) {            log.warn("putMessage message topic length too long " + msg.getTopic().length());            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);        }
//        消息属性长度不合法        if (msg.getPropertiesString() != null && msg.getPropertiesString().length() > Short.MAX_VALUE) {            log.warn("putMessage message properties length too long " + msg.getPropertiesString().length());            return new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, null);        }
//        系统繁忙        if (this.isOSPageCacheBusy()) {            return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, null);        }
        long beginTime = this.getSystemClock().now();//        commitLog存储消息=》        PutMessageResult result = this.commitLog.putMessage(msg);
        long eclipseTime = this.getSystemClock().now() - beginTime;        if (eclipseTime > 500) {            log.warn("putMessage not in lock eclipse time(ms)={}, bodyLength={}", eclipseTime, msg.getBody().length);        }        this.storeStatsService.setPutMessageEntireTimeMax(eclipseTime);
        if (null == result || !result.isOk()) {            this.storeStatsService.getPutMessageFailedTimes().incrementAndGet();        }
        return result;    }

进入这个方法，commitLog存储消息，org.apache.rocketmq.store.CommitLog#putMessage

public PutMessageResult putMessage(final MessageExtBrokerInner msg) {        // Set the storage time        msg.setStoreTimestamp(System.currentTimeMillis());        // Set the message body BODY CRC (consider the most appropriate setting        // on the client)        msg.setBodyCRC(UtilAll.crc32(msg.getBody()));        // Back to Results        AppendMessageResult result = null;
        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
//        从消息中获取topic        String topic = msg.getTopic();//        从消息中获取queueId        int queueId = msg.getQueueId();
//        获取事务类型        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());//        如果没有事务或提交事务延迟执行        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE            || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {            // Delay Delivery            if (msg.getDelayTimeLevel() > 0) {                if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) {                    msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel());                }
//                设置延迟消息的topic                topic = ScheduleMessageService.SCHEDULE_TOPIC;                queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel());
                // Backup real topic, queueId 备份真正的topic和queueId                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));                msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
                msg.setTopic(topic);                msg.setQueueId(queueId);            }        }
        long eclipseTimeInLock = 0;        MappedFile unlockMappedFile = null;//        获取映射文件队列的最后一个映射文件        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
//        自旋锁或者互斥锁        putMessageLock.lock(); //spin or ReentrantLock ,depending on store config        try {            long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();//            开始锁定时间            this.beginTimeInLock = beginLockTimestamp;
            // Here settings are stored timestamp, in order to ensure an orderly            // global            msg.setStoreTimestamp(beginLockTimestamp);
            if (null == mappedFile || mappedFile.isFull()) {//                映射文件不存在或者映射文件满了以起始位置的offset获取最后的映射文件=》                mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise            }            if (null == mappedFile) {                log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());                beginTimeInLock = 0;                return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null);            }
//            映射文件中添加消息=》            result = mappedFile.appendMessage(msg, this.appendMessageCallback);            switch (result.getStatus()) {                case PUT_OK:                    break;//                    映射文件不存在或者映射文件满了                case END_OF_FILE:                    unlockMappedFile = mappedFile;                    // Create a new file, re-write the message 创建一个文件读写消息                    mappedFile = this.mappedFileQueue.getLastMappedFile(0);                    if (null == mappedFile) {                        // XXX: warn and notify me                        log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());                        beginTimeInLock = 0;                        return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result);                    }//                    处理消息=》                    result = mappedFile.appendMessage(msg, this.appendMessageCallback);                    break;//                    消息过大                case MESSAGE_SIZE_EXCEEDED://                    消息属性过大                case PROPERTIES_SIZE_EXCEEDED:                    beginTimeInLock = 0;                    return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result);                case UNKNOWN_ERROR:                    beginTimeInLock = 0;                    return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);                default:                    beginTimeInLock = 0;                    return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);            }
            eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;            beginTimeInLock = 0;        } finally {            putMessageLock.unlock();        }
        if (eclipseTimeInLock > 500) {            log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, result);        }
        if (null != unlockMappedFile && this.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) {//            解锁映射文件            this.defaultMessageStore.unlockMappedFile(unlockMappedFile);        }
        PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);
        // Statistics 单次存储消息topic次数        storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();//        单次存储消息topic大小        storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes());
//        磁盘刷新=》        handleDiskFlush(result, putMessageResult, msg);//        主从刷新=》        handleHA(result, putMessageResult, msg);
        return putMessageResult;    }

进入这个方法，映射文件不存在或者映射文件满了以起始位置的offset获取最后的映射文件，org.apache.rocketmq.store.MappedFileQueue#getLastMappedFile(long, boolean)

public MappedFile getLastMappedFile(final long startOffset, boolean needCreate) {        long createOffset = -1;//        获取映射文件队列中最后一个映射文件        MappedFile mappedFileLast = getLastMappedFile();
        if (mappedFileLast == null) {            createOffset = startOffset - (startOffset % this.mappedFileSize);        }
        if (mappedFileLast != null && mappedFileLast.isFull()) {//            创建的offset=最后映射文件的开始offset+映射文件的大小            createOffset = mappedFileLast.getFileFromOffset() + this.mappedFileSize;        }
//        创建文件的offset不是-1且需要创建映射文件        if (createOffset != -1 && needCreate) {//            下个文件存储路径 System.getProperty("user.home") + File.separator + "store"//            + File.separator + "commitlog"，根据offset创建文件名            String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset);//            下下个文件存储路经            String nextNextFilePath = this.storePath + File.separator                + UtilAll.offset2FileName(createOffset + this.mappedFileSize);            MappedFile mappedFile = null;
            if (this.allocateMappedFileService != null) {//                处理请求返回映射文件=》                mappedFile = this.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath,                    nextNextFilePath, this.mappedFileSize);            } else {                try {//                    创建映射文件=》                    mappedFile = new MappedFile(nextFilePath, this.mappedFileSize);                } catch (IOException e) {                    log.error("create mappedFile exception", e);                }            }
            if (mappedFile != null) {                if (this.mappedFiles.isEmpty()) {                    mappedFile.setFirstCreateInQueue(true);                }                this.mappedFiles.add(mappedFile);            }
            return mappedFile;        }
        return mappedFileLast;    }

进入这个方法，处理请求返回映射文件，org.apache.rocketmq.store.AllocateMappedFileService#putRequestAndReturnMappedFile

public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String nextNextFilePath, int fileSize) {        int canSubmitRequests = 2;//        是否瞬间持久化        if (this.messageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {//            如果broker是master，buffer不够用瞬间失败            if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool()                && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool                canSubmitRequests = this.messageStore.getTransientStorePool().remainBufferNumbs() - this.requestQueue.size();            }        }
        AllocateRequest nextReq = new AllocateRequest(nextFilePath, fileSize);//        缓存存储请求        boolean nextPutOK = this.requestTable.putIfAbsent(nextFilePath, nextReq) == null;
        if (nextPutOK) {            if (canSubmitRequests <= 0) {                log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " +                    "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().remainBufferNumbs());                this.requestTable.remove(nextFilePath);                return null;            }//            下一个请求添加到优先级阻塞队列中            boolean offerOK = this.requestQueue.offer(nextReq);            if (!offerOK) {                log.warn("never expected here, add a request to preallocate queue failed");            }            canSubmitRequests--;        }
        AllocateRequest nextNextReq = new AllocateRequest(nextNextFilePath, fileSize);//        缓存下下个请求        boolean nextNextPutOK = this.requestTable.putIfAbsent(nextNextFilePath, nextNextReq) == null;        if (nextNextPutOK) {            if (canSubmitRequests <= 0) {                log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " +                    "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().remainBufferNumbs());                this.requestTable.remove(nextNextFilePath);            } else {//                下下个请求加入优先级阻塞队列                boolean offerOK = this.requestQueue.offer(nextNextReq);                if (!offerOK) {                    log.warn("never expected here, add a request to preallocate queue failed");                }            }        }
        if (hasException) {            log.warn(this.getServiceName() + " service has exception. so return null");            return null;        }
        AllocateRequest result = this.requestTable.get(nextFilePath);        try {            if (result != null) {//                同步等待                boolean waitOK = result.getCountDownLatch().await(waitTimeOut, TimeUnit.MILLISECONDS);                if (!waitOK) {                    log.warn("create mmap timeout " + result.getFilePath() + " " + result.getFileSize());                    return null;                } else {                    this.requestTable.remove(nextFilePath);                    return result.getMappedFile();                }            } else {                log.error("find preallocate mmap failed, this never happen");            }        } catch (InterruptedException e) {            log.warn(this.getServiceName() + " service has exception. ", e);        }
        return null;    }

往上返回到这个方法，映射文件中添加消息，org.apache.rocketmq.store.MappedFile#appendMessagesInner

 public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb) {        assert messageExt != null;        assert cb != null;
//        获取当前写的位置        int currentPos = this.wrotePosition.get();
        if (currentPos < this.fileSize) {            ByteBuffer byteBuffer = writeBuffer != null ? writeBuffer.slice() : this.mappedByteBuffer.slice();            byteBuffer.position(currentPos);            AppendMessageResult result = null;            if (messageExt instanceof MessageExtBrokerInner) {//                消息序列化后组装映射的buffer=》                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBrokerInner) messageExt);            } else if (messageExt instanceof MessageExtBatch) {//                批量消息序列化后组装映射的buffer                result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBatch) messageExt);            } else {                return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);            }            this.wrotePosition.addAndGet(result.getWroteBytes());            this.storeTimestamp = result.getStoreTimestamp();            return result;        }        log.error("MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}", currentPos, this.fileSize);        return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);    }

进入这个方法，消息序列化后组装映射的buffer，org.apache.rocketmq.store.CommitLog.DefaultAppendMessageCallback#doAppend(long, java.nio.ByteBuffer, int, org.apache.rocketmq.store.MessageExtBrokerInner)

public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank,            final MessageExtBrokerInner msgInner) {            // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>
            // PHY OFFSET 写的offset            long wroteOffset = fileFromOffset + byteBuffer.position();
            this.resetByteBuffer(hostHolder, 8);            String msgId = MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(hostHolder), wroteOffset);
            // Record ConsumeQueue information            keyBuilder.setLength(0);            keyBuilder.append(msgInner.getTopic());            keyBuilder.append('-');            keyBuilder.append(msgInner.getQueueId());            String key = keyBuilder.toString();//            缓存topic、queue和offset信息            Long queueOffset = CommitLog.this.topicQueueTable.get(key);            if (null == queueOffset) {                queueOffset = 0L;                CommitLog.this.topicQueueTable.put(key, queueOffset);            }
            // Transaction messages that require special handling            final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag());            switch (tranType) {                // Prepared and Rollback message is not consumed, will not enter the                // consumer queuec                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:                    queueOffset = 0L;                    break;                case MessageSysFlag.TRANSACTION_NOT_TYPE:                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:                default:                    break;            }
            /**             * Serialize message             */            final byte[] propertiesData =                msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);
            final int propertiesLength = propertiesData == null ? 0 : propertiesData.length;
//            消息属性长度过大            if (propertiesLength > Short.MAX_VALUE) {                log.warn("putMessage message properties length too long. length={}", propertiesData.length);                return new AppendMessageResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED);            }
//            获取topic数据            final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);            final int topicLength = topicData.length;
            final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;
//            计算消息长度            final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength);
            // Exceeds the maximum message            if (msgLen > this.maxMessageSize) {                CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength                    + ", maxMessageSize: " + this.maxMessageSize);                return new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED);            }
            // Determines whether there is sufficient free space            if ((msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) {                this.resetByteBuffer(this.msgStoreItemMemory, maxBlank);                // 1 TOTALSIZE                this.msgStoreItemMemory.putInt(maxBlank);                // 2 MAGICCODE                this.msgStoreItemMemory.putInt(CommitLog.BLANK_MAGIC_CODE);                // 3 The remaining space may be any value                // Here the length of the specially set maxBlank                final long beginTimeMills = CommitLog.this.defaultMessageStore.now();                byteBuffer.put(this.msgStoreItemMemory.array(), 0, maxBlank);                return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgId, msgInner.getStoreTimestamp(),                    queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);            }
            // Initialization of storage space            this.resetByteBuffer(msgStoreItemMemory, msgLen);            // 1 TOTALSIZE            this.msgStoreItemMemory.putInt(msgLen);            // 2 MAGICCODE            this.msgStoreItemMemory.putInt(CommitLog.MESSAGE_MAGIC_CODE);            // 3 BODYCRC            this.msgStoreItemMemory.putInt(msgInner.getBodyCRC());            // 4 QUEUEID            this.msgStoreItemMemory.putInt(msgInner.getQueueId());            // 5 FLAG            this.msgStoreItemMemory.putInt(msgInner.getFlag());            // 6 QUEUEOFFSET            this.msgStoreItemMemory.putLong(queueOffset);            // 7 PHYSICALOFFSET            this.msgStoreItemMemory.putLong(fileFromOffset + byteBuffer.position());            // 8 SYSFLAG            this.msgStoreItemMemory.putInt(msgInner.getSysFlag());            // 9 BORNTIMESTAMP            this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp());            // 10 BORNHOST            this.resetByteBuffer(hostHolder, 8);            this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder));            // 11 STORETIMESTAMP            this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp());            // 12 STOREHOSTADDRESS            this.resetByteBuffer(hostHolder, 8);            this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder));            //this.msgBatchMemory.put(msgInner.getStoreHostBytes());            // 13 RECONSUMETIMES            this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes());            // 14 Prepared Transaction Offset            this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset());            // 15 BODY            this.msgStoreItemMemory.putInt(bodyLength);            if (bodyLength > 0)                this.msgStoreItemMemory.put(msgInner.getBody());            // 16 TOPIC            this.msgStoreItemMemory.put((byte) topicLength);            this.msgStoreItemMemory.put(topicData);            // 17 PROPERTIES            this.msgStoreItemMemory.putShort((short) propertiesLength);            if (propertiesLength > 0)                this.msgStoreItemMemory.put(propertiesData);
            final long beginTimeMills = CommitLog.this.defaultMessageStore.now();            // Write messages to the queue buffer            byteBuffer.put(this.msgStoreItemMemory.array(), 0, msgLen);
            AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgId,                msgInner.getStoreTimestamp(), queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills);
            switch (tranType) {                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:                    break;                case MessageSysFlag.TRANSACTION_NOT_TYPE:                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:                    // The next update ConsumeQueue information                    CommitLog.this.topicQueueTable.put(key, ++queueOffset);                    break;                default:                    break;            }            return result;        }

往上返回到这个方法，磁盘刷新，org.apache.rocketmq.store.CommitLog#handleDiskFlush

 public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {        // Synchronization flush 同步刷新        if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {            final GroupCommitService service = (GroupCommitService) this.flushCommitLogService;            if (messageExt.isWaitStoreMsgOK()) {                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());                service.putRequest(request);//                countdownLatch.await() 同步等待刷新结果，除非超时                boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());                if (!flushOK) {                    log.error("do groupcommit, wait for flush failed, topic: " + messageExt.getTopic() + " tags: " + messageExt.getTags()                        + " client address: " + messageExt.getBornHostString());                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);                }            } else {//                如果异步直接解除阻塞 countdownLatch.countDown()                service.wakeup();            }        }        // Asynchronous flush 异步刷新        else {            if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {                flushCommitLogService.wakeup();            } else {                commitLogService.wakeup();            }        }    }

往上返回到这个方法，主从刷新，org.apache.rocketmq.store.CommitLog#handleHA

public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {//        如果master同步刷新        if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {            HAService service = this.defaultMessageStore.getHaService();            if (messageExt.isWaitStoreMsgOK()) {                // Determine whether to wait                if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {                    GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());                    service.putRequest(request);                    service.getWaitNotifyObject().wakeupAll();//                    countDownLatch.await 同步等待刷新，除非等待超时                    boolean flushOK =                        request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());                    if (!flushOK) {                        log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "                            + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());                        putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);                    }                }                // Slave problem                else {                    // Tell the producer, slave not available                    putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);                }            }        }
    }

下篇继续。

说在最后

本次解析仅代表个人观点，仅供参考。

加入技术微信群

钉钉技术群

转载于:https://my.oschina.net/u/3775437/blog/3097014

chucuavins01156

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
rocketmq源码解析结束事务处理器①

说在前面结束事务处理器源码解析进入这个方法，org.apache.rocketmq.broker.processor.EndTransactionProcessor#processRequest @Override public RemotingCommand pr...
复制链接

扫一扫