Hadoop NameNode之RPC Server（二）

  下面讲Handler是如何处理Call请求的，简单的说就是利用动态代理做一次函数调用，将返回值发给Responser，然后由Responser发送给客户端，Handler的处理函数如下：

public void run() {
      LOG.info(getName() + ": starting");
      SERVER.set(Server.this);
      //分配返回数据的缓冲区
      ByteArrayOutputStream buf =
        new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
      while (running) {
        try {
          final Call call = callQueue.take(); // 从调用队列中拿一个出来进行处理
          if (LOG.isDebugEnabled())
            LOG.debug(getName() + ": has #" + call.id + " from " +
                      call.connection);
         
          String errorClass = null;
          String error = null;
          Writable value = null;
 
          CurCall.set(call);
          //下面这里开始调用，返回值为value
          try {
            // Make the call as the user viaSubject.doAs, thus associating
            // the call with the Subject
            if (call.connection.user == null) {
              value = call(call.connection.protocol, call.param,
                           call.timestamp);
            } else {
              value =
                call.connection.user.doAs
                  (new PrivilegedExceptionAction<Writable>(){
                     @Override
                     public Writable run() throws Exception {
                       // 这里开始真正的调用
                       return call(call.connection.protocol,
                                   call.param, call.timestamp);
 
                     }
                   }
                  );
            }
          } catch (Throwable e) {
            LOG.info(getName()+", call "+call+": error: " + e, e);
            errorClass =e.getClass().getName();
            error = StringUtils.stringifyException(e);
          }
          CurCall.set(null);
          synchronized (call.connection.responseQueue) {
            // setupResponse() needs to be sync'ed togetherwith
            // responder.doResponse() sincesetupResponse may use
            // SASL to encrypt response data and SASLenforces
            // its own message ordering.
            setupResponse(buf, call,
                        (error == null) ? Status.SUCCESS : Status.ERROR,
                        value, errorClass,error);
          // Discard the large buf and resetit back to
          // smaller size to freeup heap
          if (buf.size() > maxRespSize) {
            LOG.warn("Large response size " + buf.size() + " for call " +
                call.toString());
              buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
            }
            responder.doRespond(call);
          }
        } catch (InterruptedException e) {
          if (running) {                          // unexpected -- log it
            LOG.info(getName() + " caught: " +
                     StringUtils.stringifyException(e));
          }
        } catch (Exception e) {
          LOG.info(getName() + " caught: " +
                   StringUtils.stringifyException(e));
        }
      }
      LOG.info(getName() + ": exiting");
    }
  }

上面代码中，Call对象的调用和返回值的设置是重头戏，这个函数里一坨的日志记录和异常捕获，我们只看关键部分，执行完后返回一个Writable类型的可序列化数据，用于返回。

先看第一个：调用

public Writablecall(Class<?> protocol, Writable param, long receivedTime)
    throws IOException {
      try {
         //这里的param是实现Writable和Invocation接口的，所以可以自由转化
        Invocation call = (Invocation)param;
        if (verbose) log("Call:" + call);
        //获得具体的调用方法，因为我们前台执行的是ls命令，所以这里的函数为getFileInfo
        Method method =
         protocol.getMethod(call.getMethodName(),
                                  call.getParameterClasses());
        method.setAccessible(true);
        //动态代理中常用的功能，记录调用时间，返回类型为Object
        long startTime = System.currentTimeMillis();
        Object value = method.invoke(instance, call.getParameters());
        int processingTime = (int) (System.currentTimeMillis() -startTime);
        int qTime = (int) (startTime-receivedTime);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Served: " + call.getMethodName() +
                    " queueTime= " + qTime +
                    " procesingTime= " + processingTime);
        }
        rpcMetrics.addRpcQueueTime(qTime);
        rpcMetrics.addRpcProcessingTime(processingTime);
        rpcMetrics.addRpcProcessingTime(call.getMethodName(),processingTime);
        if (verbose) log("Return:"+value);
        //创建返回值并返回
        return new ObjectWritable(method.getReturnType(), value);
 
      } catch (InvocationTargetException e) {
        Throwable target =e.getTargetException();
        if (target instanceof IOException) {
          throw (IOException)target;
        } else {
          IOException ioe = new IOException(target.toString());
         ioe.setStackTrace(target.getStackTrace());
          throw ioe;
        }
      } catch (Throwable e) {
        if (!(e instanceof IOException)) {
          LOG.error("Unexpected throwable object ", e);
        }
        IOException ioe = new IOException(e.toString());
        ioe.setStackTrace(e.getStackTrace());
        throw ioe;
      }
    }
  }

看返回值的处理，在上面函数执行完后，会将返回值存放在Call对象的Response成员中

private void setupResponse(ByteArrayOutputStreamresponse,
                             Call call, Statusstatus,
                             Writable rv,String errorClass, String error)
  throws IOException {
    response.reset();
DataOutputStream out = new DataOutputStream(response);
//调用编号、成功状态、错误类型一个都不能少
    out.writeInt(call.id);                // write call id
    out.writeInt(status.state);          // write status
 
if (status == Status.SUCCESS) {
  //返回值的写入
      rv.write(out);
    } else {
      WritableUtils.writeString(out,errorClass);
      WritableUtils.writeString(out,error);
    }
    if (call.connection.useWrap) {
      wrapWithSasl(response, call);
}
//复制给Call对象的成员变量，然后就剩返回了
    call.setResponse(ByteBuffer.wrap(response.toByteArray()));
  }
 

   上面函数执行完后，会进入responder.doRespond(call)中，在这里主要会调用Responser的processResponse进行数据返回的操作，调用关系不如上图来的直接，所以直接贴图了，呵呵

void doRespond(Call call) throws IOException{
      synchronized (call.connection.responseQueue) {
        //注意，这里会把call对象放入返回队列哦
        call.connection.responseQueue.addLast(call);
        if (call.connection.responseQueue.size() == 1) {
          //开始处理返回数据
          processResponse(call.connection.responseQueue, true);
        }
      }
}

再看下真正发送数据的函数

private int channelWrite(WritableByteChannel channel,
                           ByteBuffer buffer) throws IOException {
   
    int count = (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
                 channel.write(buffer) : channelIO(null, channel, buffer);
    if (count > 0) {
      rpcMetrics.incrSentBytes(count);
    }
    return count;
  }

这个函数执行完后，客户端就能看到返回数据了

上面的演示中我们看到了在发送的时候用到了Responser对象，那么为什么还要启动一个Responser线程呢？我们看下这个线程的执行体就能理解了

public void run() {
      LOG.info(getName() + ": starting");
      SERVER.set(Server.this);
      long lastPurgeTime = 0;   // last check for old calls.
 
      while (running) {
        try {
          waitPending();     // If a channel is being registered, wait.
          writeSelector.select(PURGE_INTERVAL);
          Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();
          while (iter.hasNext()) {
            SelectionKey key = iter.next();
            iter.remove();
            try {
              //这里会处理SelectionKey的OP_WRITE事件，做异步写操作，
              //其实内部调用函数是一样的,如果网络情况不佳，那么处理OP_WRITE事件
              //能有效降低网络阻塞对服务器的影响
              if (key.isValid() && key.isWritable()){
                  doAsyncWrite(key);//
              }
            } catch (IOException e) {
              LOG.info(getName() + ": doAsyncWrite threw exception " + e);
            }
          }
          long now = System.currentTimeMillis();
          if (now < lastPurgeTime + PURGE_INTERVAL) {
            continue;
          }
          lastPurgeTime = now;
          //
          // 长时间没有处理的Call会被清除
          //
          LOG.debug("Checking for old callresponses.");
          ArrayList<Call> calls;
         
          // get the list of channels from list ofkeys.
          synchronized (writeSelector.keys()) {
            calls = new ArrayList<Call>(writeSelector.keys().size());
            iter = writeSelector.keys().iterator();
            while (iter.hasNext()) {
              SelectionKey key = iter.next();
              Call call =(Call)key.attachment();
              if (call != null && key.channel() == call.connection.channel) {
                calls.add(call);
              }
            }
          }
         
          for(Call call : calls) {
            try {
              doPurge(call, now);
            } catch (IOException e) {
              LOG.warn("Error in purging old calls " + e);
            }
          }
        } catch (OutOfMemoryError e) {
          //
          // we can run out of memory if we have toomany threads
          // log the event and sleep for a minute andgive
          // some thread(s) a chance to finish
          //
          LOG.warn("Out of Memory in server select", e);
          try { Thread.sleep(60000); } catch (Exception ie) {}
        } catch (Exception e) {
          LOG.warn("Exception in Responder " +
                   StringUtils.stringifyException(e));
        }
      }
      LOG.info("Stopping " + this.getName());
    }