zookeeper的watcher源码（四）

一、客户端注册watcher

1. 引入zkClient的maven依赖
2. 客户端注册 watcher 有 3 种方式，getData、exists、getChildren；

   public class WatcherDemo {
   
       public static final String host = "122.51.167.43:2181";
       public static final String path = "/watcher";
   
       public static void main(String[] args) {
           ZooKeeper zooKeeper = null;
           try {
               zooKeeper = new ZooKeeper(host, 5000, new Watcher() {
                   @Override
                   public void process(WatchedEvent event) {
                       System.out.println(event.getType());
                   }
               });
               zooKeeper.create(path, "q".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
   
               //节点监听+节点变化
   //            zooKeeper.exists(path, true);
   //            zooKeeper.getData(path, true, null);
               zooKeeper.setData(path, "a".getBytes(), -1);
   
               //子节点监听+节点变化
               zooKeeper.getChildren(path, true);
               zooKeeper.create(path + "/subNode", "q".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
           } catch (Exception e) {
               e.printStackTrace();
           }
       }
   }
   

3. zookeeper实例化过程

   public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher,
           boolean canBeReadOnly, HostProvider aHostProvider,
           ZKClientConfig clientConfig) throws IOException {
       LOG.info("Initiating client connection, connectString=" + connectString
               + " sessionTimeout=" + sessionTimeout + " watcher=" + watcher);
   
       if (clientConfig == null) {
           clientConfig = new ZKClientConfig();
       }
       this.clientConfig = clientConfig;
       //创建ZKWatchManager管理客户端的watcher
       watchManager = defaultWatchManager();
       //将watcher赋值给ZKWatchManager的defaultWatcher 
       watchManager.defaultWatcher = watcher;
       ConnectStringParser connectStringParser = new ConnectStringParser(
               connectString);
       hostProvider = aHostProvider;
   	//ClientCnxn用于和服务端通信（SendThread）+客户端watcher事件通知（EventThread）
       cnxn = new ClientCnxn(connectStringParser.getChrootPath(),
               hostProvider, sessionTimeout, this, watchManager,
               getClientCnxnSocket(), canBeReadOnly);
       //SendThread、EventThread启动
       cnxn.start();
   }
   

二、服务端接收客户端发送的数据

1. 服务端接收客户端发送的数据，NIOServerCnxnFactory接收客户端请求，run方法里面创建一个NIOServerCnxn用于接收请求，doIO → readPayload → readRequest → 转给ZookeeperServer的processPacket方法处理

   public void processPacket(ServerCnxn cnxn, ByteBuffer incomingBuffer) throws IOException {
       // We have the request, now process and setup for next
       InputStream bais = new ByteBufferInputStream(incomingBuffer);
       BinaryInputArchive bia = BinaryInputArchive.getArchive(bais);
       RequestHeader h = new RequestHeader();
       //请求头反序列化
       h.deserialize(bia, "header");
       // Through the magic of byte buffers, txn will not be
       // pointing
       // to the start of the txn
       incomingBuffer = incomingBuffer.slice();
       //类型是不是认证类型
       if (h.getType() == OpCode.auth) {
           LOG.info("got auth packet " + cnxn.getRemoteSocketAddress());
           AuthPacket authPacket = new AuthPacket();
           ByteBufferInputStream.byteBuffer2Record(incomingBuffer, authPacket);
           String scheme = authPacket.getScheme();
           AuthenticationProvider ap = ProviderRegistry.getProvider(scheme);
           Code authReturn = KeeperException.Code.AUTHFAILED;
           if(ap != null) {
               try {
                   authReturn = ap.handleAuthentication(cnxn, authPacket.getAuth());
               } catch(RuntimeException e) {
                   LOG.warn("Caught runtime exception from AuthenticationProvider: " + scheme + " due to " + e);
                   authReturn = KeeperException.Code.AUTHFAILED;                   
               }
           }
           if (authReturn!= KeeperException.Code.OK) {
               if (ap == null) {
                   LOG.warn("No authentication provider for scheme: "
                           + scheme + " has "
                           + ProviderRegistry.listProviders());
               } else {
                   LOG.warn("Authentication failed for scheme: " + scheme);
               }
               // send a response...
               ReplyHeader rh = new ReplyHeader(h.getXid(), 0,
                       KeeperException.Code.AUTHFAILED.intValue());
               cnxn.sendResponse(rh, null, null);
               // ... and close connection
               cnxn.sendBuffer(ServerCnxnFactory.closeConn);
               cnxn.disableRecv();
           } else {
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Authentication succeeded for scheme: "
                             + scheme);
               }
               LOG.info("auth success " + cnxn.getRemoteSocketAddress());
               ReplyHeader rh = new ReplyHeader(h.getXid(), 0,
                       KeeperException.Code.OK.intValue());
               cnxn.sendResponse(rh, null, null);
           }
           return;
       } else {
           //是不是sasl
           if (h.getType() == OpCode.sasl) {
               Record rsp = processSasl(incomingBuffer,cnxn);
               ReplyHeader rh = new ReplyHeader(h.getXid(), 0, KeeperException.Code.OK.intValue());
               cnxn.sendResponse(rh,rsp, "response"); // not sure about 3rd arg..what is it?
               return;
           }
           //真正处理请求的
           else {
               Request si = new Request(cnxn, cnxn.getSessionId(), h.getXid(),
                 h.getType(), incomingBuffer, cnxn.getAuthInfo());
               si.setOwner(ServerCnxn.me);
               submitRequest(si);
           }
       }
       cnxn.incrOutstandingRequests(h);
   }
   

2. zookeeperServer的submitRequest方法

   public void submitRequest(Request si) {
       if (firstProcessor == null) {
           synchronized (this) {
               try {
                   // Since all requests are passed to the request
                   // processor it should wait for setting up the request
                   // processor chain. The state will be updated to RUNNING
                   // after the setup.
                   while (state == State.INITIAL) {
                       wait(1000);
                   }
               } catch (InterruptedException e) {
                   LOG.warn("Unexpected interruption", e);
               }
               if (firstProcessor == null || state != State.RUNNING) {
                   throw new RuntimeException("Not started");
               }
           }
       }
       try {
           touch(si.cnxn);
           //是否合法
           boolean validpacket = Request.isValid(si.type);
           if (validpacket) {
               //zk的责任链，在ZookeeperServer初始化后调用 setupRequestProcessor方法设置的
               //单机和集群ZookeeperServer的实现类不一样，所以责任链也不一样， 下面分析
               firstProcessor.processRequest(si);
               if (si.cnxn != null) {
                   incInProcess();
               }
           } else {
               LOG.warn("Received packet at server of unknown type " + si.type);
               new UnimplementedRequestProcessor().processRequest(si);
           }
       } catch (MissingSessionException e) {
           if (LOG.isDebugEnabled()) {
               LOG.debug("Dropping request: " + e.getMessage());
           }
       } catch (RequestProcessorException e) {
           LOG.error("Unable to process request:" + e.getMessage(), e);
       }
   }
   

3. 单机版 ZookeeperServer 责任链初始化

    //单机版zk
   //PrepRequestProcessor → SyncRequestProcessor → FinalRequestProcessor
   protected void setupRequestProcessors() {
       RequestProcessor finalProcessor = new FinalRequestProcessor(this);
       RequestProcessor syncProcessor = new SyncRequestProcessor(this,
               finalProcessor);
       ((SyncRequestProcessor)syncProcessor).start();
       firstProcessor = new PrepRequestProcessor(this, syncProcessor);
       ((PrepRequestProcessor)firstProcessor).start();
   }
   
   //再看 调用firstProcessor.processRequest(si); 则会如下所示
   public void processRequest(Request request) {
       // request.addRQRec(">prep="+zks.outstandingChanges.size());
       //添加到阻塞队列中，由自己的run线程去消费
       submittedRequests.add(request);
   }
   //PrepRequestProcessor 的run方法
   @Override
   public void run() {
       try {
           while (true) {
               Request request = submittedRequests.take();
               long traceMask = ZooTrace.CLIENT_REQUEST_TRACE_MASK;
               if (request.type == OpCode.ping) {
                   traceMask = ZooTrace.CLIENT_PING_TRACE_MASK;
               }
               if (LOG.isTraceEnabled()) {
                   ZooTrace.logRequest(LOG, traceMask, 'P', request, "");
               }
               if (Request.requestOfDeath == request) {
                   break;
               }
               //预处理
               pRequest(request);
           }
       } catch (RequestProcessorException e) {
           if (e.getCause() instanceof XidRolloverException) {
               LOG.info(e.getCause().getMessage());
           }
           handleException(this.getName(), e);
       } catch (Exception e) {
           handleException(this.getName(), e);
       }
       LOG.info("PrepRequestProcessor exited loop!");
   }
   protected void pRequest(Request request) throws RequestProcessorException {
       // 省略。。。
       nextProcessor.processRequest(request);
   }
   
   //nextProcessor.processRequest(request);就是SyncRequestProcessor
   public void processRequest(Request request) {
       // request.addRQRec(">sync");
       //添加到阻塞队列， 由自己线程消费
       queuedRequests.add(request);
   }
   @Override
   public void run() {
       try {
           int logCount = 0;
   
           // we do this in an attempt to ensure that not all of the servers
           // in the ensemble take a snapshot at the same time
           setRandRoll(r.nextInt(snapCount/2));
           while (true) {
               Request si = null;
               if (toFlush.isEmpty()) {
                   si = queuedRequests.take();
               } else {
                   si = queuedRequests.poll();
                   if (si == null) {
                       flush(toFlush);
                       continue;
                   }
               }
               if (si == requestOfDeath) {
                   break;
               }
               if (si != null) {
                   // track the number of records written to the log
                   //处理快照
                   if (zks.getZKDatabase().append(si)) {
                       logCount++;
                       if (logCount > (snapCount / 2 + randRoll)) {
                           setRandRoll(r.nextInt(snapCount/2));
                           // roll the log
                           zks.getZKDatabase().rollLog();
                           // take a snapshot
                           if (snapInProcess != null && snapInProcess.isAlive()) {
                               LOG.warn("Too busy to snap, skipping");
                           } else {
                               snapInProcess = new ZooKeeperThread("Snapshot Thread") {
                                       public void run() {
                                           try {
                                               zks.takeSnapshot();
                                           } catch(Exception e) {
                                               LOG.warn("Unexpected exception", e);
                                           }
                                       }
                                   };
                               snapInProcess.start();
                           }
                           logCount = 0;
                       }
                   } else if (toFlush.isEmpty()) {
                       // optimization for read heavy workloads
                       // iff this is a read, and there are no pending
                       // flushes (writes), then just pass this to the next
                       // processor
                       if (nextProcessor != null) {
                           //交给下一个处理器
                           nextProcessor.processRequest(si);
                           if (nextProcessor instanceof Flushable) {
                               ((Flushable)nextProcessor).flush();
                           }
                       }
                       continue;
                   }
                   toFlush.add(si);
                   if (toFlush.size() > 1000) {
                       flush(toFlush);
                   }
               }
           }
       } catch (Throwable t) {
           handleException(this.getName(), t);
           running = false;
       }
       LOG.info("SyncRequestProcessor exited!");
   }
   
   //最后交给FinalRequestProcessor，看之前客户端操作类型为exists
   case OpCode.exists: {
       lastOp = "EXIS";
       // TODO we need to figure out the security requirement for this!
       ExistsRequest existsRequest = new ExistsRequest();
       //反序列化获取客户端发送来的request
       ByteBufferInputStream.byteBuffer2Record(request.request,
               existsRequest);
       //请求路径
       String path = existsRequest.getPath();
       if (path.indexOf('\0') != -1) {
           throw new KeeperException.BadArgumentsException();
       }
       //组装节点信息
       //客户端请求中是否设置了watcher，并将其保存在服务端的WatchManager中的HashMap<String, HashSet<Watcher>>
       //Watcher为ServerCnxn
       Stat stat = zks.getZKDatabase().statNode(path, existsRequest
               .getWatch() ? cnxn : null);
       //封装响应客户端
       rsp = new ExistsResponse(stat);
       break;
   }
   

4. 集群版 LeaderZookeeperServer 责任链初始化

   //zk集群，选为leader节点
   //PrepRequestProcessor → ProposalRequestProcessor → CommitProcessor →  ToBeAppliedRequestProcessor → FinalRequestProcessor
   @Override
   protected void setupRequestProcessors() {
       RequestProcessor finalProcessor = new FinalRequestProcessor(this);
       RequestProcessor toBeAppliedProcessor = new Leader.ToBeAppliedRequestProcessor(
               finalProcessor, getLeader().toBeApplied);
       commitProcessor = new CommitProcessor(toBeAppliedProcessor,
               Long.toString(getServerId()), false,
               getZooKeeperServerListener());
       commitProcessor.start();
       ProposalRequestProcessor proposalProcessor = new ProposalRequestProcessor(this,
               commitProcessor);
       proposalProcessor.initialize();
       firstProcessor = new PrepRequestProcessor(this, proposalProcessor);
       ((PrepRequestProcessor)firstProcessor).start();
   }
   

5. 集群版 FollowerZookeeperServer 责任链初始化

   //zk集群，选为follower节点
   //FollowerRequestProcessor → CommitProcessor → FinalRequestProcessor
   @Override
   protected void setupRequestProcessors() {
       RequestProcessor finalProcessor = new FinalRequestProcessor(this);
       commitProcessor = new CommitProcessor(finalProcessor,
               Long.toString(getServerId()), true,
               getZooKeeperServerListener());
       commitProcessor.start();
       firstProcessor = new FollowerRequestProcessor(this, commitProcessor);
       ((FollowerRequestProcessor) firstProcessor).start();
       syncProcessor = new SyncRequestProcessor(this,
               new SendAckRequestProcessor((Learner)getFollower()));
       syncProcessor.start();
   }
   

二、客户端的SendThread接收服务端的处理完成的响应

1. 客户端的SendThread接收服务端的处理完成的响应，交给ClientCnxnSocketNIO的doIO方法处理

   void readResponse(ByteBuffer incomingBuffer) throws IOException {
           ByteBufferInputStream bbis = new ByteBufferInputStream(
                   incomingBuffer);
           BinaryInputArchive bbia = BinaryInputArchive.getArchive(bbis);
           //zk自创jute序列化工具
           ReplyHeader replyHdr = new ReplyHeader();
   		//反序列化得到数据
           replyHdr.deserialize(bbia, "header");
           //-2表示ping
           if (replyHdr.getXid() == -2) {
               // -2 is the xid for pings
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Got ping response for sessionid: 0x"
                           + Long.toHexString(sessionId)
                           + " after "
                           + ((System.nanoTime() - lastPingSentNs) / 1000000)
                           + "ms");
               }
               return;
           }
           //
           if (replyHdr.getXid() == -4) {
               // -4 is the xid for AuthPacket               
               if(replyHdr.getErr() == KeeperException.Code.AUTHFAILED.intValue()) {
                   state = States.AUTH_FAILED;                    
                   eventThread.queueEvent( new WatchedEvent(Watcher.Event.EventType.None, 
                           Watcher.Event.KeeperState.AuthFailed, null) );            		            		
               }
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Got auth sessionid:0x"
                           + Long.toHexString(sessionId));
               }
               return;
           }
           //-1是一个通知
           if (replyHdr.getXid() == -1) {
               // -1 means notification
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Got notification sessionid:0x"
                       + Long.toHexString(sessionId));
               }
               WatcherEvent event = new WatcherEvent();
               event.deserialize(bbia, "response");
   
               // convert from a server path to a client path
               if (chrootPath != null) {
                   String serverPath = event.getPath();
                   if(serverPath.compareTo(chrootPath)==0)
                       event.setPath("/");
                   else if (serverPath.length() > chrootPath.length())
                       event.setPath(serverPath.substring(chrootPath.length()));
                   else {
                   	LOG.warn("Got server path " + event.getPath()
                   			+ " which is too short for chroot path "
                   			+ chrootPath);
                   }
               }
   
               WatchedEvent we = new WatchedEvent(event);
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Got " + we + " for sessionid 0x"
                           + Long.toHexString(sessionId));
               }
   
               eventThread.queueEvent( we );
               return;
           }
   
           // If SASL authentication is currently in progress, construct and
           // send a response packet immediately, rather than queuing a
           // response as with other packets.
           if (tunnelAuthInProgress()) {
               GetSASLRequest request = new GetSASLRequest();
               request.deserialize(bbia,"token");
               zooKeeperSaslClient.respondToServer(request.getToken(),
                 ClientCnxn.this);
               return;
           }
   
           Packet packet;
           synchronized (pendingQueue) {
               if (pendingQueue.size() == 0) {
                   throw new IOException("Nothing in the queue, but got "
                           + replyHdr.getXid());
               }
               packet = pendingQueue.remove();
           }
           /*
            * Since requests are processed in order, we better get a response
            * to the first request!
            */
           try {
           	//校验数据包信息，校验成功后讲数据包信息进行更新（替换为服务端的信息）
               if (packet.requestHeader.getXid() != replyHdr.getXid()) {
                   packet.replyHeader.setErr(
                           KeeperException.Code.CONNECTIONLOSS.intValue());
                   throw new IOException("Xid out of order. Got Xid "
                           + replyHdr.getXid() + " with err " +
                           + replyHdr.getErr() +
                           " expected Xid "
                           + packet.requestHeader.getXid()
                           + " for a packet with details: "
                           + packet );
               }
   
               packet.replyHeader.setXid(replyHdr.getXid());
               packet.replyHeader.setErr(replyHdr.getErr());
               packet.replyHeader.setZxid(replyHdr.getZxid());
               if (replyHdr.getZxid() > 0) {
                   lastZxid = replyHdr.getZxid();
               }
               if (packet.response != null && replyHdr.getErr() == 0) {
              		 //获得服务端的响应，反序列化以后设置到 packet.response 属性中。所以我们可以在exists 方法的最后一行通过 packet.response 拿到改请求的返回结果
                   packet.response.deserialize(bbia, "response");
               }
   
               if (LOG.isDebugEnabled()) {
                   LOG.debug("Reading reply sessionid:0x"
                           + Long.toHexString(sessionId) + ", packet:: " + packet);
               }
           } finally {
               //客户端保存watcher
               finishPacket(packet);
           }
       }
   

2. 客户端保存watcher

   private void finishPacket(Packet p) {
       int err = p.replyHeader.getErr();
       if (p.watchRegistration != null) {
       	//ZkWatcherManager中保存
           p.watchRegistration.register(err);
       }
       // Add all the removed watch events to the event queue, so that the
       // clients will be notified with 'Data/Child WatchRemoved' event type.
       if (p.watchDeregistration != null) {
           Map<EventType, Set<Watcher>> materializedWatchers = null;
           try {
               materializedWatchers = p.watchDeregistration.unregister(err);
               for (Entry<EventType, Set<Watcher>> entry : materializedWatchers
                       .entrySet()) {
                   Set<Watcher> watchers = entry.getValue();
                   if (watchers.size() > 0) {
                       queueEvent(p.watchDeregistration.getClientPath(), err,
                               watchers, entry.getKey());
                       // ignore connectionloss when removing from local
                       // session
                       p.replyHeader.setErr(Code.OK.intValue());
                   }
               }
           } catch (KeeperException.NoWatcherException nwe) {
               p.replyHeader.setErr(nwe.code().intValue());
           } catch (KeeperException ke) {
               p.replyHeader.setErr(ke.code().intValue());
           }
       }
   
       if (p.cb == null) {
           synchronized (p) {
               p.finished = true;
               p.notifyAll();
           }
       } else {
           p.finished = true;
           //加入等待事件通知的队列中，等待服务端通知
           eventThread.queuePacket(p);
       }
   }
   

三、客户端修改数据触发服务端数据变化

1. WatcherDemo中zooKeeper.setData(path, “a”.getBytes(), -1); SendThread发送到服务端被责任链FinalRequestProcessor的processRequest方法处理，进一步调用zks.processTxn(hdr, txn); → getZKDatabase().processTxn(hdr, txn); → dataTree.processTxn(hdr, txn); → setData方法

   //节点变化
   public Stat setData(String path, byte data[], int version, long zxid,
           long time) throws KeeperException.NoNodeException {
       Stat s = new Stat();
       DataNode n = nodes.get(path);
       if (n == null) {
           throw new KeeperException.NoNodeException();
       }
       byte lastdata[] = null;
       synchronized (n) {
           lastdata = n.data;
           n.data = data;
           n.stat.setMtime(time);
           n.stat.setMzxid(zxid);
           n.stat.setVersion(version);
           n.copyStat(s);
       }
       // now update if the path is in a quota subtree.
       String lastPrefix;
       if((lastPrefix = getMaxPrefixWithQuota(path)) != null) {
         this.updateBytes(lastPrefix, (data == null ? 0 : data.length)
             - (lastdata == null ? 0 : lastdata.length));
       }
       //触发事件
       dataWatches.triggerWatch(path, EventType.NodeDataChanged);
       return s;
   }
   

2. WatchManager的 triggerWatch方法

   public Set<Watcher> triggerWatch(String path, EventType type, Set<Watcher> supress) {
       WatchedEvent e = new WatchedEvent(type,
               KeeperState.SyncConnected, path);
       HashSet<Watcher> watchers;
       synchronized (this) {
           watchers = watchTable.remove(path);
           if (watchers == null || watchers.isEmpty()) {
               if (LOG.isTraceEnabled()) {
                   ZooTrace.logTraceMessage(LOG,
                           ZooTrace.EVENT_DELIVERY_TRACE_MASK,
                           "No watchers for " + path);
               }
               return null;
           }
           for (Watcher w : watchers) {
               HashSet<String> paths = watch2Paths.get(w);
               if (paths != null) {
                   paths.remove(path);
               }
           }
       }
       for (Watcher w : watchers) {
           if (supress != null && supress.contains(w)) {
               continue;
           }
           //NIOServerCnxn的process方法
           w.process(e);
       }
       return watchers;
   }
   

3. 事件通知

   @Override
   synchronized public void process(WatchedEvent event) {
       ReplyHeader h = new ReplyHeader(-1, -1L, 0);
       if (LOG.isTraceEnabled()) {
           ZooTrace.logTraceMessage(LOG, ZooTrace.EVENT_DELIVERY_TRACE_MASK,
                                    "Deliver event " + event + " to 0x"
                                    + Long.toHexString(this.sessionId)
                                    + " through " + this);
       }
   
       // Convert WatchedEvent to a type that can be sent over the wire
       WatcherEvent e = event.getWrapper();
       //事件通知
       sendResponse(h, e, "notification");
   }
   

四、客户端处理事件通知

1. SendThread 的readResponse方法中

   // -1 是服务端事件通知
   if (replyHdr.getXid() == -1) {
      // -1 means notification
       if (LOG.isDebugEnabled()) {
           LOG.debug("Got notification sessionid:0x"
               + Long.toHexString(sessionId));
       }
       WatcherEvent event = new WatcherEvent();
       event.deserialize(bbia, "response");
   
       // convert from a server path to a client path
       if (chrootPath != null) {
           String serverPath = event.getPath();
           if(serverPath.compareTo(chrootPath)==0)
               event.setPath("/");
           else if (serverPath.length() > chrootPath.length())
               event.setPath(serverPath.substring(chrootPath.length()));
           else {
           	LOG.warn("Got server path " + event.getPath()
           			+ " which is too short for chroot path "
           			+ chrootPath);
           }
       }
   
       WatchedEvent we = new WatchedEvent(event);
       if (LOG.isDebugEnabled()) {
           LOG.debug("Got " + we + " for sessionid 0x"
                   + Long.toHexString(sessionId));
       }
   	//交给EventThread线程处理
       eventThread.queueEvent( we );
       return;
   }
   

2. EventThread线程

   @Override
    public void run() {
       try {
          isRunning = true;
          while (true) {
             Object event = waitingEvents.take();
             if (event == eventOfDeath) {
                wasKilled = true;
             } else {
               //处理事件
                processEvent(event);
             }
             if (wasKilled)
                synchronized (waitingEvents) {
                   if (waitingEvents.isEmpty()) {
                      isRunning = false;
                      break;
                   }
                }
          }
       } catch (InterruptedException e) {
          LOG.error("Event thread exiting due to interruption", e);
       }
   
        LOG.info("EventThread shut down for session: 0x{}",
                 Long.toHexString(getSessionId()));
    }
   

3. private void processEvent(Object event) {
         try {
         	  //匹配不同的事件类型
             if (event instanceof WatcherSetEventPair) {
                 // each watcher will process the event
                 WatcherSetEventPair pair = (WatcherSetEventPair) event;
                 //执行所有的watcher
                 for (Watcher watcher : pair.watchers) {
                     try {
                         watcher.process(pair.event);
                     } catch (Throwable t) {
                         LOG.error("Error while calling watcher ", t);
                     }
                 }
               } else if (event instanceof LocalCallback) {
   		//省略。。。
   }