dubbo核心之服务端接收请求过程（七）

一、服务端启动NettyServer入口

1、由dubbo服务端开启端口为20880的netty服务为入口

1、NettyServer接收客户端请求
@Override
protected void doOpen() throws Throwable {
    bootstrap = new ServerBootstrap();

    bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("NettyServerBoss", true));
    workerGroup = new NioEventLoopGroup(getUrl().getPositiveParameter(IO_THREADS_KEY, Constants.DEFAULT_IO_THREADS),
            new DefaultThreadFactory("NettyServerWorker", true));
	//全双工的Handler，可读可写
	//初始化NettyServer时候，看构造器中由HeaderExchanger的bind方法 传入包装的handler，分别为DecodeHandler(HeaderExchangeHandler(handler)),再加上NettyServer的包装，最终结构为：
	//MultiMessageHandler(HeartbeatHandler(AllChannelHandler(DecodeHandler(HeaderExchangeHandler(handler)))))
	//其中这个handler是DubboProtocol创建出来的，如果不关注这个，后续会找不到调用位置
    final NettyServerHandler nettyServerHandler = new NettyServerHandler(getUrl(), this);
    channels = nettyServerHandler.getChannels();

    bootstrap.group(bossGroup, workerGroup)
            .channel(NioServerSocketChannel.class)
            .childOption(ChannelOption.TCP_NODELAY, Boolean.TRUE)
            .childOption(ChannelOption.SO_REUSEADDR, Boolean.TRUE)
            .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            .childHandler(new ChannelInitializer<NioSocketChannel>() {
                @Override
                protected void initChannel(NioSocketChannel ch) throws Exception {
                    // FIXME: should we use getTimeout()?
                    int idleTimeout = UrlUtils.getIdleTimeout(getUrl());
                    NettyCodecAdapter adapter = new NettyCodecAdapter(getCodec(), getUrl(), NettyServer.this);
                    ch.pipeline()//.addLast("logging",new LoggingHandler(LogLevel.INFO))//for debug
                            .addLast("decoder", adapter.getDecoder())
                            .addLast("encoder", adapter.getEncoder())
                            .addLast("server-idle-handler", new IdleStateHandler(0, 0, idleTimeout, MILLISECONDS))
                            .addLast("handler", nettyServerHandler);
                }
            });
    // bind
    ChannelFuture channelFuture = bootstrap.bind(getBindAddress());
    channelFuture.syncUninterruptibly();
    channel = channelFuture.channel();

}

2、MultiMessageHandler(HeartbeatHandler(AllChannelHandler(DecodeHandler(HeaderExchangeHandler(handler)))))
//NettyServerHandler的
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
     NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
     try {
     	//处理客户端的请求
         handler.received(channel, msg); 
     } finally {
         NettyChannel.removeChannelIfDisconnected(ctx.channel());
     }
 }

3、第一层MultiMessageHandler
@Override
public void received(Channel channel, Object message) throws RemotingException {
     if (message instanceof MultiMessage) {
         MultiMessage list = (MultiMessage) message;
         //如果消息是一个list，则进行拆分
         for (Object obj : list) { 
             handler.received(channel, obj);
         }
     } else {
         handler.received(channel, message);
     }
 }

4、第二层HeartbeatHandler
@Override
public void received(Channel channel, Object message) throws RemotingException {
    setReadTimestamp(channel);
    //是不是心跳请求
    if (isHeartbeatRequest(message)) {
        Request req = (Request) message;
        //带返回值
        if (req.isTwoWay()) {
            //返回一个心跳包
            Response res = new Response(req.getId(), req.getVersion());
            res.setEvent(Response.HEARTBEAT_EVENT);
            channel.send(res);
            if (logger.isInfoEnabled()) {
                int heartbeat = channel.getUrl().getParameter(Constants.HEARTBEAT_KEY, 0);
                if (logger.isDebugEnabled()) {
                    logger.debug("Received heartbeat from remote channel " + channel.getRemoteAddress()
                            + ", cause: The channel has no data-transmission exceeds a heartbeat period"
                            + (heartbeat > 0 ? ": " + heartbeat + "ms" : ""));
                }
            }
        }
        return;
    }
    if (isHeartbeatResponse(message)) {
        if (logger.isDebugEnabled()) {
            logger.debug("Receive heartbeat response in thread " + Thread.currentThread().getName());
        }
        return;
    }
    handler.received(channel, message);
}

5、第三层AllChannelHandler
@Override
public void received(Channel channel, Object message) throws RemotingException {
    ExecutorService executor = getExecutorService();
    try {
        //构建线程来来处理请求分发
        executor.execute(new ChannelEventRunnable(channel, handler, ChannelState.RECEIVED, message));
    } catch (Throwable t) {
        //TODO A temporary solution to the problem that the exception information can not be sent to the opposite end after the thread pool is full. Need a refactoring
        //fix The thread pool is full, refuses to call, does not return, and causes the consumer to wait for time out
    	if(message instanceof Request && t instanceof RejectedExecutionException){
    		Request request = (Request)message;
    		if(request.isTwoWay()){
    			String msg = "Server side(" + url.getIp() + "," + url.getPort() + ") threadpool is exhausted ,detail msg:" + t.getMessage();
    			Response response = new Response(request.getId(), request.getVersion());
    			response.setStatus(Response.SERVER_THREADPOOL_EXHAUSTED_ERROR);
    			response.setErrorMessage(msg);
    			channel.send(response);
    			return;
    		}
    	}
        throw new ExecutionException(message, channel, getClass() + " error when process received event .", t);
    }
}

6、ChannelEventRunnable的run方法
@Override
public void run() {
	//是否是接收消息
    if (state == ChannelState.RECEIVED) {
        try {
            handler.received(channel, message);
        } catch (Exception e) {
            logger.warn("ChannelEventRunnable handle " + state + " operation error, channel is " + channel
                    + ", message is " + message, e);
        }
    } else {
        switch (state) {
        case CONNECTED:
            try {
                handler.connected(channel);
            } catch (Exception e) {
                logger.warn("ChannelEventRunnable handle " + state + " operation error, channel is " + channel, e);
            }
            break;
        case DISCONNECTED:
            try {
                handler.disconnected(channel);
            } catch (Exception e) {
                logger.warn("ChannelEventRunnable handle " + state + " operation error, channel is " + channel, e);
            }
            break;
        case SENT:
            try {
                handler.sent(channel, message);
            } catch (Exception e) {
                logger.warn("ChannelEventRunnable handle " + state + " operation error, channel is " + channel
                        + ", message is " + message, e);
            }
            break;
        case CAUGHT:
            try {
                handler.caught(channel, exception);
            } catch (Exception e) {
                logger.warn("ChannelEventRunnable handle " + state + " operation error, channel is " + channel
                        + ", message is: " + message + ", exception is " + exception, e);
            }
            break;
        default:
            logger.warn("unknown state: " + state + ", message is " + message);
        }
    }

}

7、第四层DecodeHandler
@Override
public void received(Channel channel, Object message) throws RemotingException {
    //对消息进行解码
    if (message instanceof Decodeable) {
        decode(message);
    }

    if (message instanceof Request) {
        decode(((Request) message).getData());
    }

    if (message instanceof Response) {
        decode(((Response) message).getResult());
    }

    handler.received(channel, message);
}

8、第五层HeaderExchangeHandler
@Override
public void received(Channel channel, Object message) throws RemotingException {
      channel.setAttribute(KEY_READ_TIMESTAMP, System.currentTimeMillis());
      final ExchangeChannel exchangeChannel = HeaderExchangeChannel.getOrAddChannel(channel);
      try {
          //消息类型，如果是请求消息
          if (message instanceof Request) {
              // handle request.
              Request request = (Request) message;
              if (request.isEvent()) {
                  handlerEvent(channel, request);
              } else {
              	  //twoWay有返回值
                  if (request.isTwoWay()) { 
                  	//9、
                      handleRequest(exchangeChannel, request);
                  } else {
                      handler.received(exchangeChannel, request.getData());
                  }
              }
          } 
          //响应消息
          else if (message instanceof Response) { 
              handleResponse(channel, (Response) message);
          } 
          //telnet消息在cmd里面输入的命令（）
          else if (message instanceof String) {
              if (isClientSide(channel)) {
                  Exception e = new Exception("Dubbo client can not supported string message: " + message + " in channel: " + channel + ", url: " + channel.getUrl());
                  logger.error(e.getMessage(), e);
              } else {
                  String echo = handler.telnet(channel, (String) message);
                  if (echo != null && echo.length() > 0) {
                      channel.send(echo);
                  }
              }
          } else {
              handler.received(exchangeChannel, message);
          }
      } finally {
          HeaderExchangeChannel.removeChannelIfDisconnected(channel);
      }
  }

9、 
void handleRequest(final ExchangeChannel channel, Request req) throws RemotingException {
        Response res = new Response(req.getId(), req.getVersion());
        if (req.isBroken()) {
            Object data = req.getData();

            String msg;
            if (data == null) {
                msg = null;
            } else if (data instanceof Throwable) {
                msg = StringUtils.toString((Throwable) data);
            } else {
                msg = data.toString();
            }
            res.setErrorMessage("Fail to decode request due to: " + msg);
            res.setStatus(Response.BAD_REQUEST);

            channel.send(res);
            return;
        }
        // find handler by message class.
        Object msg = req.getData();
        try {
        	//10、
        	//上面说到的，handler实例在DubboProtocol中
            CompletionStage<Object> future = handler.reply(channel, msg);
            //拿到结果封装成Response返回，到此结束
            future.whenComplete((appResult, t) -> {
                try {
                    if (t == null) {
                        res.setStatus(Response.OK);
                        res.setResult(appResult);
                    } else {
                        res.setStatus(Response.SERVICE_ERROR);
                        res.setErrorMessage(StringUtils.toString(t));
                    }
                    channel.send(res);
                } catch (RemotingException e) {
                    logger.warn("Send result to consumer failed, channel is " + channel + ", msg is " + e);
                } finally {
                    // HeaderExchangeChannel.removeChannelIfDisconnected(channel);
                }
            });
        } catch (Throwable e) {
            res.setStatus(Response.SERVICE_ERROR);
            res.setErrorMessage(StringUtils.toString(e));
            channel.send(res);
        }
    }

10、DubboProtocol中handler实例，拿到invoke反射执行目标方法拿到结果
@Override
public CompletableFuture<Object> reply(ExchangeChannel channel, Object message) throws RemotingException {

     if (!(message instanceof Invocation)) {
         throw new RemotingException(channel, "Unsupported request: "
                 + (message == null ? null : (message.getClass().getName() + ": " + message))
                 + ", channel: consumer: " + channel.getRemoteAddress() + " --> provider: " + channel.getLocalAddress());
     }

     Invocation inv = (Invocation) message;
     Invoker<?> invoker = getInvoker(channel, inv);
     // need to consider backward-compatibility if it's a callback
     if (Boolean.TRUE.toString().equals(inv.getAttachments().get(IS_CALLBACK_SERVICE_INVOKE))) {
         String methodsStr = invoker.getUrl().getParameters().get("methods");
         boolean hasMethod = false;
         if (methodsStr == null || !methodsStr.contains(",")) {
             hasMethod = inv.getMethodName().equals(methodsStr);
         } else {
             String[] methods = methodsStr.split(",");
             for (String method : methods) {
                 if (inv.getMethodName().equals(method)) {
                     hasMethod = true;
                     break;
                 }
             }
         }
         if (!hasMethod) {
             logger.warn(new IllegalStateException("The methodName " + inv.getMethodName()
                     + " not found in callback service interface ,invoke will be ignored."
                     + " please update the api interface. url is:"
                     + invoker.getUrl()) + " ,invocation is :" + inv);
             return null;
         }
     }
     RpcContext.getContext().setRemoteAddress(channel.getRemoteAddress());
     //invoker=ProtocolFilterWrapper(InvokerDelegate(DelegateProviderMetaDataInvoker(AbstractProxyInvoker)))
     Result result = invoker.invoke(inv);
     return result.completionFuture().thenApply(Function.identity());
 }

11、AbstractProxyInvoker保存了Wrapper的实现类
@Override
 public <T> Invoker<T> getInvoker(T proxy, Class<T> type, URL url) {
     // TODO Wrapper cannot handle this scenario correctly: the classname contains '$'
     final Wrapper wrapper = Wrapper.getWrapper(proxy.getClass().getName().indexOf('$') < 0 ? proxy.getClass() : type);
     return new AbstractProxyInvoker<T>(proxy, type, url) {
         @Override
         protected Object doInvoke(T proxy, String methodName,
                                   Class<?>[] parameterTypes,
                                   Object[] arguments) throws Throwable {
              //wrapper代理类，直接调用目标对象的目标方法（没有用反射）
             return wrapper.invokeMethod(proxy, methodName, parameterTypes, arguments);
         }
     };
 }