Spark隐藏接口

Spark隐藏接口

Spark有一些隐藏的API, 官方没有暴露出来, 下面是一些示例, 可以方便开发

获取job状态

curl http://spark-cluster-ip:6066/v1/submissions/status/driver-20151008145126-0000 

返回样例

 {
  "action" : "SubmissionStatusResponse",
  "driverState" : "FINISHED",
  "serverSparkVersion" : "1.5.0",
  "submissionId" : "driver-20151008145126-0000",
  "success" : true,
  "workerHostPort" : "192.168.3.153:46894",
  "workerId" : "worker-20151007093409-192.168.3.153-46894"
 }

停止job

curl -X POST http://spark-cluster-ip:6066/v1/submissions/kill/driver-20151008145126-0000 

返回样例

 {
  "action" : "KillSubmissionResponse",
  "message" : "Kill request for driver-20151008145126-0000 submitted",
  "serverSparkVersion" : "1.5.0",
  "submissionId" : "driver-20151008145126-0000",
  "success" : true
 }

提交job

curl -X POST http://spark-cluster-ip:6066/v1/submissions/create --header "Content-Type:application/json;charset=UTF-8" --data '{
  "action" : "CreateSubmissionRequest",
  "appArgs" : [ "myAppArgument1" ],
  "appResource" : "file:/myfilepath/spark-job-1.0.jar",
  "clientSparkVersion" : "1.5.0",
  "environmentVariables" : {
    "SPARK_ENV_LOADED" : "1"
  },
  "mainClass" : "com.mycompany.MyJob",
  "sparkProperties" : {
    "spark.jars" : "file:/myfilepath/spark-job-1.0.jar",
    "spark.driver.supervise" : "false",
    "spark.app.name" : "MyJob",
    "spark.eventLog.enabled": "true",
    "spark.submit.deployMode" : "cluster",
    "spark.master" : "spark://spark-cluster-ip:6066"
  }
}'  

返回样例

{
  "action" : "CreateSubmissionResponse",
  "message" : "Driver successfully submitted as driver-20151008145126-0000",
  "serverSparkVersion" : "1.5.0",
  "submissionId" : "driver-20151008145126-0000",
  "success" : true
}

Apache Spark Hidden REST API

Raw

 get_job_status.sh

curl http://spark-cluster-ip:6066/v1/submissions/status/driver-20151008145126-0000

Raw

 get_job_status_result.json

	{
	"action" : "SubmissionStatusResponse",
	"driverState" : "FINISHED",
	"serverSparkVersion" : "1.5.0",
	"submissionId" : "driver-20151008145126-0000",
	"success" : true,
	"workerHostPort" : "192.168.3.153:46894",
	"workerId" : "worker-20151007093409-192.168.3.153-46894"
	}

Raw

 kill_job.sh

curl -X POST http://spark-cluster-ip:6066/v1/submissions/kill/driver-20151008145126-0000

Raw

 kill_job_result.json

	{
	"action" : "KillSubmissionResponse",
	"message" : "Kill request for driver-20151008145126-0000 submitted",
	"serverSparkVersion" : "1.5.0",
	"submissionId" : "driver-20151008145126-0000",
	"success" : true
	}

Raw

 submit_job.sh

	curl -X POST http://spark-cluster-ip:6066/v1/submissions/create --header "Content-Type:application/json;charset=UTF-8" --data '{
	"action" : "CreateSubmissionRequest",
	"appArgs" : [ "myAppArgument1" ],
	"appResource" : "file:/myfilepath/spark-job-1.0.jar",
	"clientSparkVersion" : "1.5.0",
	"environmentVariables" : {
	"SPARK_ENV_LOADED" : "1"
	},
	"mainClass" : "com.mycompany.MyJob",
	"sparkProperties" : {
	"spark.jars" : "file:/myfilepath/spark-job-1.0.jar",
	"spark.driver.supervise" : "false",
	"spark.app.name" : "MyJob",
	"spark.eventLog.enabled": "true",
	"spark.submit.deployMode" : "cluster",
	"spark.master" : "spark://spark-cluster-ip:6066"
	}
	}'

Raw

 submit_job_result.json

	{
	"action" : "CreateSubmissionResponse",
	"message" : "Driver successfully submitted as driver-20151008145126-0000",
	"serverSparkVersion" : "1.5.0",
	"submissionId" : "driver-20151008145126-0000",
	"success" : true
	}