二、zookeeper源码(Client)

这篇只说客户端，服务端后边会肝出来的哈！我们开始吧。

从zkCli.sh看源码

我们在启动客户端的时候都需要在zk的bin目录下启动zkCli.sh。那我们就先看看这个脚本里边都是什么东西。

下边是执行脚本后的样子：

chendongdong@chendongdongdeMacBook-Pro bin % ./zkCli.sh
Connecting to localhost:2181
Welcome to ZooKeeper!

WATCHER::

WatchedEvent state:SyncConnected type:None path:null
[zk: localhost:2181(CONNECTED) 0] 

ZOOBIN="${BASH_SOURCE-$0}"
ZOOBIN="$(dirname "${ZOOBIN}")"
ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)"
if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then
  . "$ZOOBINDIR"/../libexec/zkEnv.sh
else
  . "$ZOOBINDIR"/zkEnv.sh
fi
ZOO_LOG_FILE=zookeeper-$USER-cli-$HOSTNAME.log
"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \
     -cp "$CLASSPATH" $CLIENT_JVMFLAGS $JVMFLAGS \
     org.apache.zookeeper.ZooKeeperMain "$@"

开始肝源码

其实启动这个脚本也就是调用了org.apache.zookeeper.ZooKeeperMain的main方法。我们就看看代码吧。

public static void main(String args[]) throws KeeperException, IOException, InterruptedException{
  //创建ZooKeeper客户端
  ZooKeeperMain main = new ZooKeeperMain(args);
  //接收输入
  main.run();
}
public ZooKeeperMain(String args[]) throws IOException, InterruptedException {
  cl.parseOptions(args);
  System.out.println("Connecting to " + cl.getOption("server"));
  connectToZK(cl.getOption("server"));
  //zk = new ZooKeeper(cl.getOption("server"),
  //Integer.parseInt(cl.getOption("timeout")), new MyWatcher());
}
protected void connectToZK(String newHost) throws InterruptedException, IOException {
  if (zk != null && zk.getState().isAlive()) {
    zk.close();
  }
  host = newHost;
  boolean readOnly = cl.getOption("readonly") != null;
  zk = new ZooKeeper(host,Integer.parseInt(cl.getOption("timeout")),new MyWatcher(), readOnly);
}

这个就是ZooKeeperMain.main方法调用代码，我们应该可以看到最后其实执行脚本和我们用zookeeper客户端一样也是都创建了客户端一样就是创建new ZooKeeper()对象来对zk进行操作。

现在可以看看new ZooKeeper()都做了什么？

public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher,boolean canBeReadOnly) throws IOException{
  LOG.info("Initiating client connection, connectString=" + connectString
           + " sessionTimeout=" + sessionTimeout + " watcher=" + watcher);
  watchManager.defaultWatcher = watcher;
  //因为服务端地址可以存在多个都是以','隔开的。所以这里是对服务器地址的封装。
  ConnectStringParser connectStringParser = new ConnectStringParser(connectString);
  //这里是把服务地址随机返回一个。Collections.shuffle(this.serverAddresses)主要就是执行了这个方法
  HostProvider hostProvider = new StaticHostProvider(connectStringParser.getServerAddresses());
  //封装ClientCnxn对象
  cnxn = new ClientCnxn(connectStringParser.getChrootPath(),hostProvider, sessionTimeout, this, watchManager,getClientCnxnSocket(), canBeReadOnly);
  //启动zk最终要的两个线程SendThread,EventThread。启动两个线程
  cnxn.start();
}

public ClientCnxn(String chrootPath, HostProvider hostProvider, int sessionTimeout, ZooKeeper zooKeeper,ClientWatchManager watcher, ClientCnxnSocket clientCnxnSocket,long sessionId, byte[] sessionPasswd, boolean canBeReadOnly) {
  //连接基本参数
  this.zooKeeper = zooKeeper;
  this.watcher = watcher;
  this.sessionId = sessionId;
  this.sessionPasswd = sessionPasswd;
  this.sessionTimeout = sessionTimeout;
  this.hostProvider = hostProvider;
  this.chrootPath = chrootPath;
  //计算连接时间等
  connectTimeout = sessionTimeout / hostProvider.size();
  readTimeout = sessionTimeout * 2 / 3;
  readOnly = canBeReadOnly;
  //SendThread 发送packt,EventThread 处理事件这两个线程是核心
  sendThread = new SendThread(clientCnxnSocket);
  eventThread = new EventThread();
}

这里看名字SendThread,EventThread是线程,我们来证实下。

//SendThread,EventThread都集成ZooKeeperThread,并且ZooKeeperThread集成Thread所以这两个是线程,run方法则是重点。
SendThread extends ZooKeeperThread
EventThread extends ZooKeeperThread
ZooKeeperThread extends Thread

那么这两个线程是在哪里启动的呢？看上边代码cnxn.start();在创建ZooKeeper启动了这两个线程。

public void start() {
  sendThread.start();
  eventThread.start();
}

创建客户端先到这里，具体线程做了什么后续会说到哈。我们现在说说Main方法中创建客户端后的main.run();方法都干了什么？

@SuppressWarnings("unchecked")
void run() throws KeeperException, IOException, InterruptedException {
  if (cl.getCommand() == null) {
    System.out.println("Welcome to ZooKeeper!");
    boolean jlinemissing = false;
    try {
      //省略部分代码，我们看重点。
      Method readLine = consoleC.getMethod("readLine", String.class);
      //这里就是获取控制台的输入，我们来看看getPrompt()方法
      while ((line = (String)readLine.invoke(console, getPrompt())) != null) {
        //接收控制台输入执行逻辑。
        executeLine(line);
      }
    } 
      //省略部分代码，我们看重点。
}

while()循环中的这个方法是不是格外的熟悉，这就是控制台输出的提示其中commandCount是执行的数量，每次都会++的。

protected String getPrompt() {
	return "[zk: " + host + "("+zk.getState()+")" + " " + commandCount + "] ";
}

下边看重点executeLine(line)

public void executeLine(String line) throws InterruptedException, IOException, KeeperException {
  if (!line.equals("")) {
    //分解输入成zk命令
    cl.parseCommand(line);
    //加到历史
    addToHistory(commandCount,line);
    //我们要看的重点方法
    processCmd(cl);
    //执行次数++
    commandCount++;
  }
}

protected boolean processCmd(MyCommandOptions co) throws KeeperException, IOException, InterruptedException
{
  try {
    return processZKCmd(co);
  } 
  //省略异常代码
  return false;
}

 protected boolean processZKCmd(MyCommandOptions co) throws KeeperException, IOException, InterruptedException
 {
   Stat stat = new Stat();
   //省略部分代码，主要就是各种if来判断输入的是什么然后做响应的处理。
   if (cmd.equals("create") && args.length >= 3) {
     
   } else if (cmd.equals("get") && args.length >= 2) {
     //这里我们就以"get"命令来看看，其他的可以自己看看源码。
     path = args[1];
     //我们可以看到最终还是用zk客户端调用了getData方法来获取节点下的值。
     byte data[] = zk.getData(path, watch, stat);
     data = (data == null)? "null".getBytes() : data;
     System.out.println(new String(data));
     printStat(stat);
   } 
   return watch;
 }

这里总结下上边的源码都做了什么？

当我们启动zkCli.sh命令其实就是创建了ZooKeeper客户端。

new ZooKeeper()

1. 创建默认事件监听管理器
2. 设置默认服务端连接负载均衡
3. 创建ClientCnxn实例(这个实力就是和服务端socket传输数据的实例)
4. 启动ClientCnxn实例中两个线程sendThread,eventThread

main.run()

1. 循环获取命令行
2. 分解输入命令
3. 加入历史命令
4. 根据命令调用zk客户端指定api

下边我们该说说客户端怎么操作的？我们从zk.getData(path, watch, stat)作为切入点开始。

public byte[] getData(final String path, Watcher watcher, Stat stat) throws KeeperException, InterruptedException
{
  final String clientPath = path;
  //校验path
  PathUtils.validatePath(clientPath);
  //封装watch，watch是没有传递给服务端的，这里包装进去在后边在watchmanger中删除用。
  WatchRegistration wcb = null;
  if (watcher != null) {
    wcb = new DataWatchRegistration(watcher, clientPath);
  }
  final String serverPath = prependChroot(clientPath);
  //包装请求数据
  RequestHeader h = new RequestHeader();
  h.setType(ZooDefs.OpCode.getData);
  GetDataRequest request = new GetDataRequest();
  request.setPath(serverPath);
  request.setWatch(watcher != null);
  GetDataResponse response = new GetDataResponse();
  //发送请求
  ReplyHeader r = cnxn.submitRequest(h, request, response, wcb);
  return response.getData();
}

public ReplyHeader submitRequest(RequestHeader h, Record request,Record response, WatchRegistration watchRegistration) throws InterruptedException {
  ReplyHeader r = new ReplyHeader();
  //发送请求
  Packet packet = queuePacket(h, r, request, response, null, null, null,null, watchRegistration);
  //阻塞当前packet，后边返回响应后悔唤醒的。
  synchronized (packet) {
    while (!packet.finished) {
      packet.wait();
    }
  }
  return r;
}

Packet queuePacket(RequestHeader h, ReplyHeader r, Record request,Record response, AsyncCallback cb, String clientPath,String serverPath, Object ctx, WatchRegistration watchRegistration)
{
  Packet packet = null;
  synchronized (outgoingQueue) {
    //封装成packet
    packet = new Packet(h, r, request, response, watchRegistration);
    packet.cb = cb;
    packet.ctx = ctx;
    packet.clientPath = clientPath;
    packet.serverPath = serverPath;
    if (!state.isAlive() || closing) {
      conLossPacket(packet);
    } else {
      if (h.getType() == OpCode.closeSession) {
        closing = true;
      }
      //加入outgoingQueue队列
      outgoingQueue.add(packet);
    }
  }
  //唤醒发送线程
  sendThread.getClientCnxnSocket().wakeupCnxn();
  return packet;
}

以上源码可以用一句话总结：把请求数据封装成packet加入到队列唤醒sendThread线程。可以没有看到socket传递到服务端啊？

我们想想是不是忘记了上边开启的线程sendThread，我们来看看sendThread线程的run方法。一定不会失望的，也是核心逻辑。

public void run() {
  //省略部分代码
	while (state.isAlive()) {
    //是否已连接，无则连接
  	if (!clientCnxnSocket.isConnected()) {
      startConnect(serverAddress);
      clientCnxnSocket.updateLastSendAndHeard();
    }
    //已连接，是否需要发送心跳
    if (state.isConnected()) {
      int timeToNextPing = readTimeout / 2 - clientCnxnSocket.getIdleSend() - 
        ((clientCnxnSocket.getIdleSend() > 1000) ? 1000 : 0);
      if (timeToNextPing <= 0 || clientCnxnSocket.getIdleSend() > MAX_SEND_PING_INTERVAL) {
        //发送心跳
        sendPing();
        //更新下次发送心跳时间
        clientCnxnSocket.updateLastSend();
      } else {
        if (timeToNextPing < to) {
          to = timeToNextPing;
        }
      }
    }
    //发送数据
    /*
     * ClientCnxnSocket的实现
		 * ClientCnxnSocketNIO：默认使用
		 * ClientCnxnSocketNetty
     */
    clientCnxnSocket.doTransport(to, pendingQueue, outgoingQueue, ClientCnxn.this);
	} 
}

void doTransport(int waitTimeOut, List<Packet> pendingQueue, LinkedList<Packet> outgoingQueue,ClientCnxn cnxn) throws IOException, InterruptedException {
  Set<SelectionKey> selected;
  synchronized (this) {
      selected = selector.selectedKeys();
  }
  //NIO选择器,知识盲区。
  for (SelectionKey k : selected) {
      SocketChannel sc = ((SocketChannel) k.channel());
      if ((k.readyOps() & SelectionKey.OP_CONNECT) != 0) {
          if (sc.finishConnect()) {
              updateLastSendAndHeard();
              updateSocketAddresses();
              sendThread.primeConnection();
          }
      } else if ((k.readyOps() & (SelectionKey.OP_READ | SelectionKey.OP_WRITE)) != 0) {
        	//核心在于doIO方法，进入doIO方法的条件是可读或者可写，也就是该方法处理读写事件
          doIO(pendingQueue, cnxn);
      }
  }
}

我们先看写事件

if (sockKey.isWritable()) {
  synchronized(outgoingQueue) {
    //从队列中拿到一个packet。这个就是在queuePacket入队的packet
    Packet p = findSendablePacket(outgoingQueue,cnxn.sendThread.clientTunneledAuthenticationInProgress());
    if (p != null) {
      updateLastSend();
      //创建buffer，将请求头、请求体写入buffer
      if (p.bb == null) {
        if ((p.requestHeader != null) &&
            (p.requestHeader.getType() != OpCode.ping) &&
            (p.requestHeader.getType() != OpCode.auth)) {
          p.requestHeader.setXid(cnxn.getXid());
        }
        p.createBB();
      }
      //向socket写入数据
      sock.write(p.bb);
      if (!p.bb.hasRemaining()) {
        sentCount++;
        outgoingQueue.removeFirstOccurrence(p);
        if (p.requestHeader != null
            && p.requestHeader.getType() != OpCode.ping
            && p.requestHeader.getType() != OpCode.auth) {
          synchronized (pendingQueue) {
            //入队到pendingQueue，已经发送并等待响应的packet
            pendingQueue.add(p);
          }
        }
      }
    }
  }
}

我们再看读事件

if (sockKey.isReadable()) {
  //省略代码就是把响应的字节读取响应到incomingBuffer中
  //处理响应
  sendThread.readResponse(incomingBuffer);
}

void readResponse(ByteBuffer incomingBuffer) throws IOException {
  ByteBufferInputStream bbis = new ByteBufferInputStream(incomingBuffer);
  BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis);
  ReplyHeader replyHdr = new ReplyHeader();
	//反序列化响应头
  replyHdr.deserialize(bbia, "header");
  //处理ping响应
  if (replyHdr.getXid() == -2) {
    // -2 is the xid for pings
    if (LOG.isDebugEnabled()) {
      LOG.debug("Got ping response for sessionid: 0x");
    }
    return;
  }
  //处理身份验证响应
  if (replyHdr.getXid() == -4) {
    // -4 is the xid for AuthPacket               
    if(replyHdr.getErr() == KeeperException.Code.AUTHFAILED.intValue()) {
      state = States.AUTH_FAILED;                    
      eventThread.queueEvent( new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState.AuthFailed, null) );            		            		
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("Got auth sessionid:0x" + Long.toHexString(sessionId));
    }
    return;
  }
  //处理事件响应
  if (replyHdr.getXid() == -1) {
    // -1 means notification
    if (LOG.isDebugEnabled()) {
      LOG.debug("Got notification sessionid:0x" + Long.toHexString(sessionId));
    }
    WatcherEvent event = new WatcherEvent();
    event.deserialize(bbia, "response");

    // convert from a server path to a client path
    if (chrootPath != null) {
      String serverPath = event.getPath();
      if(serverPath.compareTo(chrootPath)==0)
        event.setPath("/");
      else if (serverPath.length() > chrootPath.length())
        event.setPath(serverPath.substring(chrootPath.length()));
      else {
        LOG.warn("Got server path ");
      }
    }
    WatchedEvent we = new WatchedEvent(event);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Got " + we + " for sessionid 0x" + Long.toHexString(sessionId));
    }
    eventThread.queueEvent( we );
    return;
  }
  
  if (clientTunneledAuthenticationInProgress()) {
    GetSASLRequest request = new GetSASLRequest();
    request.deserialize(bbia,"token");
    zooKeeperSaslClient.respondToServer(request.getToken(),ClientCnxn.this);
    return;
  }

  Packet packet;
  synchronized (pendingQueue) {
    if (pendingQueue.size() == 0) {
      throw new IOException("Nothing in the queue, but got "+ replyHdr.getXid());
    }
    packet = pendingQueue.remove();
  }
  //处理普通响应
  try {
    if (packet.requestHeader.getXid() != replyHdr.getXid()) {
      packet.replyHeader.setErr(KeeperException.Code.CONNECTIONLOSS.intValue());
      throw new IOException("Xid out of order. Got Xid ");
    }

    packet.replyHeader.setXid(replyHdr.getXid());
    packet.replyHeader.setErr(replyHdr.getErr());
    packet.replyHeader.setZxid(replyHdr.getZxid());
    if (replyHdr.getZxid() > 0) {
      lastZxid = replyHdr.getZxid();
    }
    if (packet.response != null && replyHdr.getErr() == 0) {
      packet.response.deserialize(bbia, "response");
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Reading reply sessionid:0x" + Long.toHexString(sessionId) + ", packet:: " + packet);
    }
  } finally {
    //注册监听器以及其他对响应数据的处理
    finishPacket(packet);
  }
}

private void finishPacket(Packet p) {
  //注册监听器
  if (p.watchRegistration != null) {
    p.watchRegistration.register(p.replyHeader.getErr());
  }

  if (p.cb == null) {
    synchronized (p) {
      //之前在submit的时候把packet给wait后就阻塞在那个死循环了。所以这里改变了循环条件finished，notifyAll唤醒了packet
      p.finished = true;
      p.notifyAll();
    }
  } else {
    p.finished = true;
    eventThread.queuePacket(p);
  }
}

public void register(int rc) {
  if (shouldAddWatch(rc)) {
    //重点说下getWatches是抽象方法。由于watchRegistration实例是在创建Packet包的时候创建的也就是取watchManager的dataWatches，在watchManager中，有多个watcher map，将不同类型的watcher添加到不同的map保存。
    Map<String, Set<Watcher>> watches = getWatches(rc);
    synchronized(watches) {
      Set<Watcher> watchers = watches.get(clientPath);
      if (watchers == null) {
        watchers = new HashSet<Watcher>();
        watches.put(clientPath, watchers);
      }
      watchers.add(watcher);
    }
  }
}

private final Map<String, Set<Watcher>> dataWatches =
    new HashMap<String, Set<Watcher>>();
private final Map<String, Set<Watcher>> existWatches =
    new HashMap<String, Set<Watcher>>();
private final Map<String, Set<Watcher>> childWatches =
    new HashMap<String, Set<Watcher>>();

总结下sendThread的run方法都做了什么。

1. 当客户端和服务端保持存活，一直循环
2. 判断socket连接是否连接，没有连接则创建连接
3. ping服务端是否通畅
4. 发送packet
   1. 读
      1. 反序列化响应头
      2. 响应为事件类型，交给eventThread处理，将触发监听器
      3. 反序列化响应体
      4. 注册监听器
      5. 变更packet状态，唤醒packet
   2. 写
      1. 创建buff，把请求头请求体写入buff，通过socket发送到服务端

接下来我们来看看Watcher是怎么处理的？

我们在读事件中看到有一段是处理Watcher的，我们单独copy看看。

//处理事件响应
if (replyHdr.getXid() == -1) {
 	//watcher入队
  eventThread.queueEvent( we );
  return;
}

public void queueEvent(WatchedEvent event) {
  if (event.getType() == EventType.None && sessionState == event.getState()) {
    return;
  }
  sessionState = event.getState();

  //从WatcherManeger中的单个类型WatcherMap中找到指定Watcher然后包装成WatcherSetEventPair
  //watcher.materialize我们看看这个方法，就可以知道为什么watcher是一次性的。
  WatcherSetEventPair pair = new WatcherSetEventPair(watcher.materialize(event.getState(), event.getType(),event.getPath()),event);
	//入队
  waitingEvents.add(pair);
}

public Set<Watcher> materialize(Watcher.Event.KeeperState state,Watcher.Event.EventType type,String clientPath) {
  Set<Watcher> result = new HashSet<Watcher>();
  switch (type) {
		//省略部分代码
    case NodeDataChanged:
    case NodeCreated:
      synchronized (dataWatches) {
        //我们可以看到他是从WatcherManeger的WatcherMap先移除指定Watcher后再把Watcher赋值给result
        addTo(dataWatches.remove(clientPath), result);
      }
      synchronized (existWatches) {
        addTo(existWatches.remove(clientPath), result);
      }
      break;
		//省略部分代码
  }
  return result;
}

现在WatcherSetEventPair入队列了，那么肯定是EventThread的run方法对入队的进行消费了。我们开看看是否是这样的？

public void run() {
  try {
    isRunning = true;
    while (true) {
      //出队操作
      Object event = waitingEvents.take();
      if (event == eventOfDeath) {
        wasKilled = true;
      } else {
        //执行回调
        processEvent(event);
      }
    }
  } catch (InterruptedException e) {
    LOG.error("Event thread exiting due to interruption", e);
  }
}

private void processEvent(Object event) {
  try {
    if (event instanceof WatcherSetEventPair) {
      WatcherSetEventPair pair = (WatcherSetEventPair) event;
      for (Watcher watcher : pair.watchers) {
        try {
          //回调watcher的process方法
          watcher.process(pair.event);
        } catch (Throwable t) {
          LOG.error("Error while calling watcher ", t);
        }
      }
    }
	}
}

总结一下Watcher机制是怎么实现的。

1. 在指定api会注册watcher到本地WatcherManger
2. 在sendThread.run中读取响应的时候会从本地WatcherManger移除并且入队到EventThread的队列中
3. EventThread.run对取出队列数据进行watcher回调
   最后附上流程图！