spark 2.0 RPC Example

先创建Endpoint和服务，可以传入相加，相减操作。

import org.apache.spark.rpc._
import org.apache.spark._
import org.apache.spark.util.{ RpcUtils, ThreadUtils }

class TestMathMaster(
    var driverEndpoint: RpcEndpointRef) {

  def testAdd(a: Int, b: Int): Int = {
    driverEndpoint.askWithRetry[Int](TestAdd(a, b))
  }
  
  def testSub(a: Int, b: Int): Int = {
    driverEndpoint.askWithRetry[Int](TestSub(a, b))
  }
}

object TestMathMaster {
  val DRIVER_ENDPOINT_NAME = "TestMathMaster"
}

sealed class TestMathBase 
case class TestAdd(a: Int, b: Int) extends TestMathBase
case class TestSub(a: Int, b: Int) extends TestMathBase
class TestMathEndpoint(
  override val rpcEnv: RpcEnv)
    extends ThreadSafeRpcEndpoint {
  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
    case TestAdd(a: Int, b: Int) =>
      System.out.println("receive TestAdd:" + a + ", b :" + b);
      context.reply((a + b))
    case TestSub(a: Int, b: Int) =>
      System.out.println("receive TestSub:" + a + ", b :" + b);
      context.reply((a - b))
  }
}

服务器端程序。

import org.apache.spark.rpc._
import org.apache.spark._
import org.apache.spark.util.{ RpcUtils, ThreadUtils }
import java.util.concurrent.TimeUnit;
object TestMathServer {
  def main(args: Array[String]) = {
    val conf = new SparkConf();
    val systemName = "sparkDriver"
    val hostname = "172.20.110.199"
    val port = 4040;
   
    val securityManager = new SecurityManager(conf)
    val rpcEnv = RpcEnv.create(systemName, hostname, port, conf, securityManager, false)
 

    val testMathMaster =
      new TestMathMaster(rpcEnv.setupEndpoint(TestMathMaster.DRIVER_ENDPOINT_NAME, new TestMathEndpoint(rpcEnv)))
    //ref.testMaster("houzhizhen")
    val result = testMathMaster.testAdd(3, 4)
    println("the result of test: " + result)
    TimeUnit.SECONDS.sleep(10000);
  }
}

客户端程序。

import org.apache.spark.rpc._
import org.apache.spark._
import org.apache.spark.util.{ RpcUtils, ThreadUtils }
import java.util.concurrent.TimeUnit;
object estMathClient {
	def main(args: Array[String]) = {
		    val conf = new SparkConf();
		    println("TestRpcEnv");
		    val isDriver = false
		    val systemName = "sparkDriver"
		    val hostname = "172.20.110.199"
		    val port = 4040;
		    val isLocal = true;
		    val securityManager = new SecurityManager(conf)
		    val rpcEnv = RpcEnv.create(systemName, hostname, port, conf, securityManager, clientMode = true)
		    val testMathMaster =
		      new TestMathMaster(rpcEnv.setupEndpointRef(RpcAddress(hostname, 4040), TestMathMaster.DRIVER_ENDPOINT_NAME))
		    val result = testMathMaster.testAdd(1, 2)
		    println("the result of test: " + result)

		    TimeUnit.SECONDS.sleep(10000);
		  }
}