RocketMQ高可用源码笔记

主从复制

RocketMQ，消息到达主节点，会同步到从节点。如果主节点宕机，Client可以在从节点拉取消息。

HAService中有三个重要的类：

• AcceptSocketService ：Listens to slave connections to create HAConnection.监听从节点的连接并创建HAConnection
• GroupTransferService : 主从同步通知实现类
• HAClient : HA Client端实现类

// org.apache.rocketmq.store.DefaultMessageStore#start : 
if (!messageStoreConfig.isEnableDLegerCommitLog()) {
   
    // DefaultMessageStore的start方法
    this.haService.start();
    this.handleScheduleMessageService(messageStoreConfig.getBrokerRole());
}

// HaService.start方法
// org.apache.rocketmq.store.ha.HAService#start
public void start() throws Exception {
   
    // 主节点 初始化配置 开始接受从节点连接
    this.acceptSocketService.beginAccept();
    // 启动三个线程用于高可用
    this.acceptSocketService.start();
    this.groupTransferService.start();
    this.haClient.start();
}

AcceptSocketService

---

org.apache.rocketmq.store.ha.HAService#start 中启动的三个线程分别是：

// 服务线程
class AcceptSocketService extends ServiceThread {
   
    private final SocketAddress socketAddressListen;
    private ServerSocketChannel serverSocketChannel;
    private Selector selector;

    public AcceptSocketService(final int port) {
   
        this.socketAddressListen = new InetSocketAddress(port);
    }

    /**
     * Starts listening to slave connections.
     *
     * @throws Exception If fails.
     */
    public void beginAccept() throws Exception {
   
        // 创建 ServerSocketChannel
        this.serverSocketChannel = ServerSocketChannel.open();
        // 创建 selector
        this.selector = RemotingUtil.openSelector();
        // 端口复用
        this.serverSocketChannel.socket().setReuseAddress(true);
        // 绑定端口
        this.serverSocketChannel.socket().bind(this.socketAddressListen);
        // 非阻塞
        this.serverSocketChannel.configureBlocking(false);
        // 监听 请求连接 事件
        this.serverSocketChannel.register(this.selector, SelectionKey.OP_ACCEPT);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void run() {
   
        log.info(this.getServiceName() + " service started");
		// stopped is volatile
        while (!this.isStopped()) {
   
            try {
   
                // 此处填入的 1000 是为了能够及时判断 stopped的状态
                // 避免 stopped is true 时还长时间在此等待
                this.selector.select(1000);
                Set<SelectionKey> selected = this.selector.selectedKeys();

                if (selected != null) {
   
                    for (SelectionKey k : selected) {
   
                        if ((k.readyOps() & SelectionKey.OP_ACCEPT) != 0) {
   
                            // 拿到 SocketChannel
                            SocketChannel sc = ((ServerSocketChannel) k.channel()).accept();

                            if (sc != null) {
   


                                try {
   
                                    // 拿到HAConnection
                                    HAConnection conn = new HAConnection(HAService.this, sc);
                                    conn.start();
                                    // 添加 HAConnection 到 connectionList
                                    HAService.this.addConnection(conn);
                                } catch (Exception e) {
   
                                    sc.close();
                                }
                            }
                        } else {
   
                            log.warn("Unexpected ops in select " + k.readyOps());
                        }
                    }

                    selected.clear();
                }
            } catch (Exception e) {
   
                log.error(this.getServiceName() + " service has exception.", e);
            }
        }
		
        log.info(this.getServiceName() + " service end");
    }

    // 省略 shutdown 和 getServiceName 方法
}

可以看出AcceptSocketService主要职责就是根据SocketChannel创建 HAConnection，然后放入connectionList中。

GroupTransferService

---

GroupTransferService:

// 服务线程
class GroupTransferService extends ServiceThread {
   

    private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject();
    private volatile List<CommitLog.GroupCommitRequest> requestsWrite = new ArrayList<>();
    private volatile List<CommitLog.GroupCommitRequest> requestsRead = new ArrayList<>();

    public synchronized void putRequest(final CommitLog.GroupCommitRequest request) {
   
        synchronized (this.requestsWrite) {
   
            this.requestsWrite.add(request);
        }
        if (hasNotified.compareAndSet(false, true)) {
   
            waitPoint.countDown(); // notify
        }
    }

    public void notifyTransferSome() {
   
        this.notifyTransferObject.wakeup();
    }

    private void swapRequests() {
   
        List<CommitLog.GroupCommitRequest> tmp = this.requestsWrite;
        this.requestsWrite = this.requestsRead;
        this.requestsRead = tmp;
    }

    private void doWaitTransfer() {
   
        synchronized (this.requestsRead) {
   
            // 不为空才进入循环
            if (!this.requestsRead.isEmpty()) {
   
                // 循环处理请求
                for (CommitLog.GroupCommitRequest req : this.requestsRead) {
   
                    boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
                    for (int i = 0; !transferOK && i < 5; i++) {
   
                        this.notifyTransferObject.waitForRunning(1000);
                        // 循环判断是否传输完成
                        transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset();
                    }

                    if (!transferOK) {
   
                        log.warn("transfer messsage to slave timeout, " + req.getNextOffset());
                    }
					// 传输给从节点成功后，唤醒 等待传输结果的 消息发送者线程
                    req.wakeupCustomer(transferOK);
                }

                this.requestsRead.clear();
            }
        }
    }

    public void run() {
   
        while (!this.isStopped()) {
   
            // waitForRunning调用onWaitEnd
            this.waitForRunning(10);
            this.doWaitTransfer();
        }
    }
    
    @Override
    protected void onWaitEnd() {
   
        // 互换 requestsWrite 和 requestsRead
        this.swapRequests();
    }

}

从以上代码可得知，GroupTransferService就是为了在传输完成后通知消息发送者线程。有两个list，一个用于读，一个用于写。每次循环互换两个list，获得requestsRead List，对requestsRead中的req遍历，得出每个req是否被传输完成，如果全部传输完成，则唤醒发送者线程；如果在限定时间内没有传输完成，则发送者线程会超时唤醒。doWaitTransfer最后会清空requestRead，之后再与requestWrite交换，requestWrite会被外部调用放入req。这样就

public static class GroupCommitRequest {
   
    private final long nextOffset;
    private final CountDownLatch countDownLatch = new CountDownLatch(1);
    private volatile boolean flushOK = false;

    public GroupCommitRequest(long nextOffset) {
   
        this.nextOffset = nextOffset;
    }

    public long getNextOffset() {
   
        return nextOffset;
    }
	
    public void wakeupCustomer(final boolean flushOK) {
   
        this.flushOK = flushOK;
        // countDown
        this.countDownLatch.countDown();
    }
	
    // 等待线程调用
    public boolean waitForFlush(long timeout) {
   
        try {
   
            this.countDownLatch.