Spark分布式消息发送流程

Spark集群中涉及分布式节点之间的通信，例如Worker启动注册信息到Master，这个过程就有消息的传输，接下来将要分析一下消息的传输过程。首先Worker的启动还是从main方法开始，之后运行期初始化OnStart方法调用registerWithMaster完成Worker向Master的注册，registerWithMaster经过一系列调用最后调用org.apache.spark.deploy.worker.Worker#registerWithMaster方法开始真正向Master发起注册信息：

  /** 向master注册worker
    *
    * @param masterEndpoint
    */
  private def registerWithMaster(masterEndpoint: RpcEndpointRef): Unit = {
    //RegisterWorker是向Master发送的消息
    masterEndpoint.ask[RegisterWorkerResponse](RegisterWorker(workerId, host, port, self, cores, memory, workerWebUiUrl))
      .onComplete {
        // This is a very fast action so we can use "ThreadUtils.sameThread"
        case Success(msg) =>
          Utils.tryLogNonFatalError {
            handleRegisterResponse(msg)
          }
        case Failure(e) =>
          logError(s"Cannot register with master: ${masterEndpoint.address}", e)
          System.exit(1)
      }(ThreadUtils.sameThread)
  }

接下来调用org.apache.spark.rpc.RpcEndpointRef#ask方法发送消息：

 def ask[T: ClassTag](message: Any): Future[T] = ask(message, defaultAskTimeout)

接下来调用org.apache.spark.rpc.netty.NettyRpcEndpointRef#ask方法：

 override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = {
    //this 是NettyRpcEndpointRef的引用，为了让消息接收到容易知道消息发送者
    nettyEnv.ask(RequestMessage(nettyEnv.address, this, message), timeout)
  }

在调用org.apache.spark.rpc.netty.NettyRpcEnv#ask方法：

  private[netty] def ask[T: ClassTag](message: RequestMessage, timeout: RpcTimeout): Future[T] = {
		//省略部分代码
    try {
      if (remoteAddr == address) {
        val p = Promise[Any]()
        p.future.onComplete {
          case Success(response) => onSuccess(response)
          case Failure(e) => onFailure(e)
        }(ThreadUtils.sameThread)
        //本地消息
        dispatcher.postLocalMessage(message, p)
      } else {
        //外部节点消息
        val rpcMessage = RpcOutboxMessage(serialize(message),
          onFailure,
          (client, response) => onSuccess(deserialize[Any](client, response)))
        //发送消息
        postToOutbox(message.receiver, rpcMessage)
        promise.future.onFailure {
          case _: TimeoutException => rpcMessage.onTimeout()
          case _ =>
        }(ThreadUtils.sameThread)
      }
		//省略部分代码
  }

接下来调用org.apache.spark.rpc.netty.NettyRpcEnv#postToOutbox方法：

  private def postToOutbox(receiver: NettyRpcEndpointRef, message: OutboxMessage): Unit = {
    if (receiver.client != null) {
      //TODO 使用当前接受者的endpointref的TransportClient传输消息
      message.sendWith(receiver.client)
    } else {
      require(receiver.address != null,
        "Cannot send message to client endpoint with no listen address.")
      val targetOutbox = {
        //TODO RpcAddress和Outbox的映射
        val outbox = outboxes.get(receiver.address)
        if (outbox == null) {
          val newOutbox = new Outbox(this, receiver.address)
          val oldOutbox = outboxes.putIfAbsent(receiver.address, newOutbox)
          if (oldOutbox == null) {
            newOutbox
          } else {
            oldOutbox
          }
        } else {
          outbox
        }
      }
      if (stopped.get) {
        // It's possible that we put `targetOutbox` after stopping. So we need to clean it.
        outboxes.remove(receiver.address)
        targetOutbox.stop()
      } else {
        targetOutbox.send(message)
      }
    }
  }

接下来调用org.apache.spark.rpc.netty.OutboxMessage#sendWith方法。再调用org.apache.spark.network.client.TransportClient#send方法进行消息发送，TransportClient的send方法实现为：

  public void send(ByteBuffer message) {
    channel.writeAndFlush(new OneWayMessage(new NioManagedBuffer(message)));
  }

到此就完成消息的发送了，通过查看TransportClient代码我们可以发现有其他传输数据的方法，TransportClient充当通信的客户端。