RocketMQ源码分析之消息主从同步（上）

主从同步消息入口代码在handleHA(result, putMessageResult, msg);主要分为异步主从和同步主从，分别由BrokerRole枚举的ASYNC_MASTER和SYNC_MASTER决定，判断是否需要等待消息在从服务器保存成功，也就是完全保证消息在主从都要保存成功，判断是否有从服务存活即连接数是否大于0，继续判断主从消息之间的偏移量是否小于haSlaveFallbehindMax = 1024 * 1024 * 256，只有这样，才可以证明从服务可用，否则直接返回从服务器不可用。同样的，构造提交请求，直接唤醒等待有消息可写的写线程即HAConnection中的WriteSocketService，等待被唤醒后返回。

public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
        HAService service = this.defaultMessageStore.getHaService();
        if (messageExt.isWaitStoreMsgOK()) {
            // Determine whether to wait
            if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
                service.putRequest(request);
                service.getWaitNotifyObject().wakeupAll();
                boolean flushOK =
                    request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                if (!flushOK) {
                    log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                        + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                }
            }
            // Slave problem
            else {
                // Tell the producer, slave not available
                putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
            }
        }
    }
}
 public boolean isSlaveOK(final long masterPutWhere) {
    boolean result = this.connectionCount.get() > 0;
    result =
        result
            && ((masterPutWhere - this.push2SlaveMaxOffset.get()) < this.defaultMessageStore
            .getMessageStoreConfig().getHaSlaveFallbehindMax());
    return result;
}

 

主broker

使用HAService来监听端口，AcceptSocketService是专门用来接收从连接请求的，然后针对每个从服务连接建立一个HAConnection专门对接。GroupTransferService类主要起到针对请求在线程之间等待通知的作用

public HAService(final DefaultMessageStore defaultMessageStore) throws IOException {
    this.defaultMessageStore = defaultMessageStore;
    this.acceptSocketService =
        new AcceptSocketService(defaultMessageStore.getMessageStoreConfig().getHaListenPort());
    this.groupTransferService = new GroupTransferService();
    this.haClient = new HAClient();
}

当服务端启动时，接收线程会开始监听接收请求，端口号为10912

public void start() throws Exception {
    this.acceptSocketService.beginAccept();
    this.acceptSocketService.start();
    this.groupTransferService.start();
    this.haClient.start();
}
publicvoid beginAccept() throws Exception {
    this.serverSocketChannel = ServerSocketChannel.open();
    this.selector = RemotingUtil.openSelector();
    this.serverSocketChannel.socket().setReuseAddress(true);
    this.serverSocketChannel.socket().bind(this.socketAddressListen);
    this.serverSocketChannel.configureBlocking(false);
    this.serverSocketChannel.register(this.selector, SelectionKey.OP_ACCEPT);
}

定时扫描看有没有从服务进行连接，如果有的话就创建HAConnection对接，然后启动后加入到连接集合中，并对连接数量加一。

public void run() {
    log.info(this.getServiceName() + " service started");

    while (!this.isStopped()) {
        try {
            this.selector.select(1000);
            Set<SelectionKey> selected = this.selector.selectedKeys();

            if (selected != null) {
                for (SelectionKey k : selected) {
                    if ((k.readyOps() & SelectionKey.OP_ACCEPT) != 0) {
                        SocketChannel sc = ((ServerSocketChannel) k.channel()).accept();

                        if (sc != null) {
                            HAService.log.info("HAService receive new connection, "
                                + sc.socket().getRemoteSocketAddress());

                            try {
                                HAConnection conn = new HAConnection(HAService.this, sc);
                                conn.start();
                                HAService.this.addConnection(conn);
                            } catch (Exception e) {
                                log.error("new HAConnection exception", e);
                                sc.close();
                            }
                        }
                    } else {
                        log.warn("Unexpected ops in select " + k.readyOps());
                    }
                }

                selected.clear();
            }
        } catch (Exception e) {
            log.error(this.getServiceName() + " service has exception.", e);
        }
    }

    log.info(this.getServiceName() + " service end");
}
public void addConnection(final HAConnection conn) {
    synchronized (this.connectionList) {
        this.connectionList.add(conn);
    }
}

连接服务，这里会分别创建读写服务于从服务进行通信，

public HAConnection(final HAService haService, final SocketChannel socketChannel) throws IOException {
    this.haService = haService;
    this.socketChannel = socketChannel;
    this.clientAddr = this.socketChannel.socket().getRemoteSocketAddress().toString();
    this.socketChannel.configureBlocking(false);
    this.socketChannel.socket().setSoLinger(false, -1);
    this.socketChannel.socket().setTcpNoDelay(true);
    this.socketChannel.socket().setReceiveBufferSize(1024 * 64);
    this.socketChannel.socket().setSendBufferSize(1024 * 64);
    this.writeSocketService = new WriteSocketService(this.socketChannel);
    this.readSocketService = new ReadSocketService(this.socketChannel);
    this.haService.getConnectionCount().incrementAndGet();
}

读服务线程只监听通道读事件

public ReadSocketService(final SocketChannel socketChannel) throws IOException {
    this.selector = RemotingUtil.openSelector();
    this.socketChannel = socketChannel;
    this.socketChannel.register(this.selector, SelectionKey.OP_READ);
    this.thread.setDaemon(true);
}

写服务线程只监听通道写事件

public WriteSocketService(final SocketChannel socketChannel) throws IOException {
    this.selector = RemotingUtil.openSelector();
    this.socketChannel = socketChannel;
    this.socketChannel.register(this.selector, SelectionKey.OP_WRITE);
    this.thread.setDaemon(true);
}

 

从broker

使用HAClient与主进行通信，当然HAClient的初始化也需要依赖从服务这边的HAService，初始化结束后就是启动线程haClient.start()

public HAClient() throws IOException {
    this.selector = RemotingUtil.openSelector();
}

判断线程状态是否有效，然后判断是否已经与主建立了连接通道，如果没建立的话，就需要获取主地址开始建立连接，并注册监听读事件

public void run() {
    log.info(this.getServiceName() + " service started");

    while (!this.isStopped()) {
        try {
            if (this.connectMaster()) {

                if (this.isTimeToReportOffset()) {
                    boolean result = this.reportSlaveMaxOffset(this.currentReportedOffset);
                    if (!result) {
                        this.closeMaster();
                    }
                }

                this.selector.select(1000);

                boolean ok = this.processReadEvent();
                if (!ok) {
                    this.closeMaster();
                }

                if (!reportSlaveMaxOffsetPlus()) {
                    continue;
                }

                long interval =
                    HAService.this.getDefaultMessageStore().getSystemClock().now()
                        - this.lastWriteTimestamp;
                if (interval > HAService.this.getDefaultMessageStore().getMessageStoreConfig()
                    .getHaHousekeepingInterval()) {
                    log.warn("HAClient, housekeeping, found this connection[" + this.masterAddress
                        + "] expired, " + interval);
                    this.closeMaster();
                    log.warn("HAClient, master not response some time, so close connection");
                }
            } else {
                this.waitForRunning(1000 * 5);
            }
        } catch (Exception e) {
            log.warn(this.getServiceName() + " service has exception. ", e);
            this.waitForRunning(1000 * 5);
        }
    }

    log.info(this.getServiceName() + " service end");
}
private boolean connectMaster() throws ClosedChannelException {
    if (null == socketChannel) {
        String addr = this.masterAddress.get();
        if (addr != null) {

            SocketAddress socketAddress = RemotingUtil.string2SocketAddress(addr);
            if (socketAddress != null) {
                this.socketChannel = RemotingUtil.connect(socketAddress);
                if (this.socketChannel != null) {
                    this.socketChannel.register(this.selector, SelectionKey.OP_READ);
                }
            }
        }

        this.currentReportedOffset = HAService.this.defaultMessageStore.getMaxPhyOffset();

        this.lastWriteTimestamp = System.currentTimeMillis();
    }

    return this.socketChannel != null;
}

这个主地址的来源有两个，一个是BrokerController初始化时当本机是从broker时并且消息存储配置中有设置主地址的话就直接通过下面的代码一步步更新。

if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
    this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
    this.updateMasterHAServerAddrPeriodically = false;
} else {
    this.updateMasterHAServerAddrPeriodically = true;
}

public void updateHaMasterAddress(String newAddr) {
    this.haService.updateMasterAddress(newAddr);
}

 public void updateMasterAddress(final String newAddr) {
    if (this.haClient != null) {
        this.haClient.updateMasterAddress(newAddr);
    }
}

public void updateMasterAddress(final String newAddr) {
    String currentAddr = this.masterAddress.get();
    if (currentAddr == null || !currentAddr.equals(newAddr)) {
        this.masterAddress.set(newAddr);
        log.info("update master address, OLD: " + currentAddr + " NEW: " + newAddr);
    }
}

否则的话就是在broker启动并向配置中心注册时，如果本机是从的话，响应中就会带着主broker的地址信息，同时也会更新同步服务slaveSynchronize的地址。

if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) {
    this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr());
}

this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr());

建立连接之后，从就会查询并保存自己这边最新的消息偏移量

public long getMaxPhyOffset() {
    return this.commitLog.getMaxOffset();
}
public long getMaxOffset() {
    return this.mappedFileQueue.getMaxOffset();
}
public long getMaxOffset() {
    MappedFile mappedFile = getLastMappedFile();
    if (mappedFile != null) {
        return mappedFile.getFileFromOffset() + mappedFile.getReadPosition();
    }
    return 0;
}

判断是否到了该给主报告偏移量的时间了，间隔为haSendHeartbeatInterval = 1000 * 5

private boolean isTimeToReportOffset() {
    long interval =
        HAService.this.defaultMessageStore.getSystemClock().now() - this.lastWriteTimestamp;
    boolean needHeart = interval > HAService.this.defaultMessageStore.getMessageStoreConfig()
        .getHaSendHeartbeatInterval();

    return needHeart;
}

如果时间到了的话就给主报告偏移量，然后就开始等待主同步消息。

private boolean reportSlaveMaxOffset(final long maxOffset) {
    this.reportOffset.position(0);
    this.reportOffset.limit(8);
    this.reportOffset.putLong(maxOffset);
    this.reportOffset.position(0);
    this.reportOffset.limit(8);

    for (int i = 0; i < 3 && this.reportOffset.hasRemaining(); i++) {
        try {
            this.socketChannel.write(this.reportOffset);
        } catch (IOException e) {
            log.error(this.getServiceName()
                + "reportSlaveMaxOffset this.socketChannel.write exception", e);
            return false;
        }
    }

    return !this.reportOffset.hasRemaining();
}