rocketMq 集群同步双写和异步复制源码实现

前言

rocketMq集群环境下多mater多slave时主从同步没有使用netty而是原生socket,具体是怎么实现的呢,在这做一个分析,之前也做过同步刷盘和异步刷盘的源码解析下面给出链接rocketMq刷盘源码解析

一、主从同步类结构

• HAService里面包含三个内部类AcceptSocketService、GroupTransferService、HAClient
• AcceptSocketService 是master的socket的服务端使用了selector模式
• HAClient 是slave的,主要用来做同步数据上报和接收master的同步数据
• GroupTransferService 检查是否同步完成,且唤醒主线程

• HAConnection里面维护者和客户端的SocketChannel,且有两个内部类WriteSocketService、ReadSocketService
• WriteSocketService主要用来向slave写数据的
• ReadSocketService主要用来接收salve的同步进度

二、同步双写

在设置为同步双写的时候,master每写入一条消息都会同步到slave当中。

• 优点：数据与服务都无单点，Master宕机情况下，消息无延迟，服务可用性与数据可用性都非常高
• 缺点：性能比异步复制模式略低，大约低 10%左右，发送单个消息的 RT
  会略高。目前主宕机后，备机不能自动切换为主机，后续会支持自动切换功能。

2.1 CommitLog.handleHA

当master执行了同步刷盘和异步刷盘后会做同步写slave的操作

 public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
   
//        如果master同步双写
        if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
   
            HAService service = this.defaultMessageStore.getHaService();
            if (messageExt.isWaitStoreMsgOK()) {
   
                // 判断和slave的连接是否大于0且master的写入位置不能大于slave 256m
                if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
   
                    GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
                    service.putRequest(request);
                    //唤醒所有WriteSocketService线程，一个master有几个slave的话就有几个连接一个连接对应一个WriteSocketService
                    service.getWaitNotifyObject().wakeupAll();
//                    countDownLatch.await 同步等待刷新，除非等待超时
                    boolean flushOK =
                        request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                    if (!flushOK) {
   
                        log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                            + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                        putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                    }
                }
                // Slave problem
                else {
   
                    // Tell the producer, slave not available
                    putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
                }
            }
        }

    }

2.2 GroupTransferService

    class GroupTransferService extends ServiceThread {
   

        private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject();
        private volatile List<CommitLog.GroupCommitRequest> requestsWrite = new ArrayList<>();
        private volatile List<CommitLog.GroupCommitRequest> requestsRead = new ArrayList<>();
//
        public synchronized void putRequest(final CommitLog.GroupCommitRequest request) {
   
            synchronized (this.requestsWrite) {
   
                this.requestsWrite.add(request);
            }
            if (hasNotified.compareAndSet(false, true)) {
   
            // 唤醒线程(waitPoint执行了await的线程)就是GroupTransferService
                waitPoint.countDown(); // notify
            }
        }
        //slave上报进度的时候会调用这
        public void notifyTransferSome() {
   
            this.notifyTransferObject.wakeup();
        }

        private void swapRequests() {
   
            List<CommitLog.GroupCommitRequest> tmp = this.requestsWrite;
            this.requestsWrite = this.requestsRead;
            this.requestsRead = tmp;
        }

        private void doWaitTransfer() {
   
            synchronized (this.requestsRead) {
   
                if (!this.requestsRead.isEmpty()) {
   
                    for (CommitLog.GroupCommitRequest req : this.requestsRead) {
   
                        boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
                        //如果WriteSocketService没同步完循环等待5次
                        for (int i = 0; !transferOK && i < 5; i++) {
   
                            // 当前线程等待1s 如果有数据不等待
                            this.notifyTransferObject.waitForRunning(1000);
                            transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
                        }

                        if (!transferOK) {
   
                            log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
                        }
                        // 同步完 唤醒主线程
                        req.wakeupCustomer(transferOK);
                    }

                    this.requestsRead.clear();
                }
            }
        }
        
		@Override
        public void run() {
   
            log.info(this.getServiceName() + " service started");

            while (!this.isStopped()) {
   
                try {
   
                    // 定时任务 等待10 毫秒，如果有数据就不等待
                    this.waitForRunning(10);
                    // 等待数据传输给slave，在这里面判断是否同步完，同步完就唤醒主线程
                    this.doWaitTransfer();
                } catch (Exception e) {
   
                    log.warn(this.getServiceName() + " service has exception. ", e);
                }
            }

            log.info(this.getServiceName() + " service end");
        }

        @Override
        protected void onWaitEnd() {
   
            this.swapRequests();
        }

        @Override
        public String getServiceName() {
   
            return GroupTransferService.class.getSimpleName();
        }
    }

2.3 WriteSocketService

    class WriteSocketService extends ServiceThread {
   
        private final Selector selector;
        private final SocketChannel socketChannel;

        private final int headerSize = 8 + 4;
        private final ByteBuffer byteBufferHeader = ByteBuffer.allocate(headerSize)