Zookeeper源码分析-单机模式服务端启动

1. 执行流程概述

单机模式的ZK服务端逻辑写在ZooKeeperServerMain类中，由⾥⾯的main函数启动，整个过程如下：

单机模式的委托启动类为： ZooKeeperServerMain

2. 服务端启动过程

看下ZooKeeperServerMain⾥⾯的main函数代码：

public static void main(String[] args) {
    ZooKeeperServerMain main = new ZooKeeperServerMain();
    main.initializeAndRun(args);
}
protected void initializeAndRun(String[] args)
throws ConfigException, IOException, AdminServerException
{
    ServerConfig config = new ServerConfig();
    //如果⼊参只有⼀个，则认为是配置⽂件的路径
    if (args.length == 1) {
    config.parse(args[0]);
    } else {
    //否则是各个参数
    config.parse(args);
    }
    runFromConfig(config);
}
//省略部分代码，只保留了核⼼逻辑
public void runFromConfig(ServerConfig config)
        throws IOException, AdminServerException {
    LOG.info("Starting server");
    FileTxnSnapLog txnLog = null;
    try {
        txnLog = new FileTxnSnapLog(config.dataLogDir, config.dataDir);

        // 初始化zkServer对象
        final ZooKeeperServer zkServer = new ZooKeeperServer(txnLog,
                config.tickTime, config.minSessionTimeout, config.maxSessionTimeout, null);

        // 服务结束钩子，用于知道服务器错误或关闭状态更改。
        final CountDownLatch shutdownLatch = new CountDownLatch(1);
        zkServer.registerServerShutdownHandler(
                new ZooKeeperServerShutdownHandler(shutdownLatch));
        // 创建admin服务，用于接收请求(创建jetty服务)
        adminServer = AdminServerFactory.createAdminServer();
        // 设置zookeeper服务
        adminServer.setZooKeeperServer(zkServer);
        // AdminServer是3.5.0之后支持的特性,启动了一个jettyserver,默认端口是8080,访问此端口可以获取Zookeeper运行时的相关信息
        adminServer.start();
        boolean needStartZKServer = true;
        // ---启动ZooKeeperServer
        // 判断配置文件中 clientportAddress是否为null
        if (config.getClientPortAddress() != null) {
            // ServerCnxnFactory是Zookeeper中的重要组件,负责处理客户端与服务器的连接
            // 初始化server端IO对象，默认是NIOServerCnxnFactory:Java原生NIO处理网络IO事件
            cnxnFactory = ServerCnxnFactory.createFactory();

            // 初始化配置信息
            cnxnFactory.configure(config.getClientPortAddress(), config.getMaxClientCnxns(), false);

            // 启动服务:此方法除了启动ServerCnxnFactory,还会启动ZooKeeper
            cnxnFactory.startup(zkServer);
            needStartZKServer = false;
        }
        if (config.getSecureClientPortAddress() != null) {
            secureCnxnFactory = ServerCnxnFactory.createFactory();
            secureCnxnFactory.configure(config.getSecureClientPortAddress(), config.getMaxClientCnxns(), true);
            secureCnxnFactory.startup(zkServer, needStartZKServer);
        }

        // 定时清除容器节点
        //container ZNodes是3.6版本之后新增的节点类型，Container类型的节点会在它没有子节点时
        // 被删除（新创建的Container节点除外），该类就是用来周期性的进行检查清理工作
        containerManager = new ContainerManager(zkServer.getZKDatabase(), zkServer.firstProcessor,
                Integer.getInteger("znode.container.checkIntervalMs", (int) TimeUnit.MINUTES.toMillis(1)),
                Integer.getInteger("znode.container.maxPerMinute", 10000)
        );
        containerManager.start();
        // 省略关闭逻辑
    } catch (InterruptedException e) {
        // warn, but generally this is ok
        LOG.warn("Server interrupted", e);
    } finally {
        if (txnLog != null) {
            txnLog.close();
        }
    }
}

可以看到关键点在于解析配置跟启动两个⽅法，先来看下解析配置逻辑，对应上⾯的configure⽅法：

public void configure(InetSocketAddress addr, int maxcc, boolean secure) throws IOException {
        if (secure) {
            throw new UnsupportedOperationException("SSL isn't supported in NIOServerCnxn");
        }

        configureSaslLogin();

        maxClientCnxns = maxcc;

        //会话超时时间
        sessionlessCnxnTimeout = Integer.getInteger(
            ZOOKEEPER_NIO_SESSIONLESS_CNXN_TIMEOUT, 10000);
        //过期队列
        cnxnExpiryQueue =
            new ExpiryQueue<NIOServerCnxn>(sessionlessCnxnTimeout);
        expirerThread = new ConnectionExpirerThread();

        //根据CPU个数计算selector线程的数量
        int numCores = Runtime.getRuntime().availableProcessors();
        // 32 cores sweet spot seems to be 4 selector threads
        numSelectorThreads = Integer.getInteger(
            ZOOKEEPER_NIO_NUM_SELECTOR_THREADS,
            Math.max((int) Math.sqrt((float) numCores/2), 1));
        if (numSelectorThreads < 1) {
            throw new IOException("numSelectorThreads must be at least 1");
        }

        //计算woker线程的数量
        numWorkerThreads = Integer.getInteger(
            ZOOKEEPER_NIO_NUM_WORKER_THREADS, 2 * numCores);

        //worker线程关闭时间
        workerShutdownTimeoutMS = Long.getLong(
            ZOOKEEPER_NIO_SHUTDOWN_TIMEOUT, 5000);

        LOG.info("Configuring NIO connection handler with "
                 + (sessionlessCnxnTimeout/1000) + "s sessionless connection"
                 + " timeout, " + numSelectorThreads + " selector thread(s), "
                 + (numWorkerThreads > 0 ? numWorkerThreads : "no")
                 + " worker threads, and "
                 + (directBufferBytes == 0 ? "gathered writes." :
                    ("" + (directBufferBytes/1024) + " kB direct buffers.")));

        //初始化selector线程
        for(int i=0; i<numSelectorThreads; ++i) {
            selectorThreads.add(new SelectorThread(i));
        }

        this.ss = ServerSocketChannel.open();
        ss.socket().setReuseAddress(true);
        LOG.info("binding to port " + addr);
        ss.socket().bind(addr);
        ss.configureBlocking(false);
        //初始化accept线程，这里看出accept线程只有一个，里面会注册监听ACCEPT事件
        acceptThread = new AcceptThread(ss, addr, selectorThreads);
    }

再来看下启动逻辑：

public void startup(ZooKeeperServer zkServer) throws IOException, InterruptedException {
        startup(zkServer, true);
}

public void startup(ZooKeeperServer zks, boolean startServer)
        throws IOException, InterruptedException {
    // 启动相关线程
    start();
    setZooKeeperServer(zks);

    //启动服务
    if (startServer) {
        // 加载数据到zkDataBase
        zks.startdata();
        // 启动定时清除session的管理器,注册jmx,添加请求处理器
        zks.startup();
    }
}

// start()方法
public void start() {
    stopped = false;
    //初始化worker线程池
    if (workerPool == null) {
        workerPool = new WorkerService(
            "NIOWorker", numWorkerThreads, false);
    }

    //挨个启动Selector线程(处理客户端请求线程),
    for(SelectorThread thread : selectorThreads) {
        if (thread.getState() == Thread.State.NEW) {
            thread.start();
        }
    }
    // ensure thread is started once and only once
    //启动acceptThread线程(处理接收连接进行事件)
    if (acceptThread.getState() == Thread.State.NEW) {
        acceptThread.start();
    }

   // ExpirerThread（处理过期连接）
    if (expirerThread.getState() == Thread.State.NEW) {
        expirerThread.start();
    }
}

// startdata()方法
public void startdata()
throws IOException, InterruptedException {
    //初始化ZKDatabase，该数据结构用来保存ZK上面存储的所有数据
    //check to see if zkDb is not null
    if (zkDb == null) {
        //初始化数据数据，这里会加入一些原始节点，例如/zookeeper
        zkDb = new ZKDatabase(this.txnLogFactory);
    }
    //加载磁盘上已经存储的数据，如果有的话
    if (!zkDb.isInitialized()) {
        loadData();
    }
}

// 启动剩余项目
public synchronized void startup() {
    //初始化session追踪器
    if (sessionTracker == null) {
    createSessionTracker();
    }
    //启动session追踪器
    startSessionTracker();
    //建⽴请求处理链路
    setupRequestProcessors();
    registerJMX();
    setState(State.RUNNING);
    notifyAll();
}

// setupRequestProcessors()建立请求处理链路
//这⾥可以看出，单机模式下请求的处理链路为：
//PrepRequestProcessor -> SyncRequestProcessor -> FinalRequestProcessor
protected void setupRequestProcessors() {
    RequestProcessor finalProcessor = new FinalRequestProcessor(this);
    RequestProcessor syncProcessor = new SyncRequestProcessor(this,
    finalProcessor);
    ((SyncRequestProcessor)syncProcessor).start();
    firstProcessor = new PrepRequestProcessor(this, syncProcessor);
    ((PrepRequestProcessor)firstProcessor).start();
}

 总结一下：

1. 注册jmx
2. 解析ServerConfig配置对象
3. 根据配置对象，运⾏单机zk服务
4. 创建管理事务⽇志和快照FileTxnSnapLog对象， zookeeperServer对象，并设置zkServer的统计对象
 5. 设置zk服务钩⼦，原理是通过设置CountDownLatch，调⽤ZooKeeperServerShutdownHandler的handle⽅法，可以将触发shutdownLatch.await⽅法继续执⾏，即调⽤shutdown关闭单机服务
6. 基于jetty创建zk的admin服务
7. 创建连接对象cnxnFactory和secureCnxnFactory（安全连接才创建该对象），⽤于处理客户端的请求 
8. 创建定时清除容器节点管理器，⽤于处理容器节点下不存在⼦节点的清理容器节点⼯作等