可以通过要监控的任务列表,对比存活的Spark任务,不存在的可以通过短信或邮件的方式预警。获取Spark还在运行状态任务的示例代码:
private def yarn(): Unit = {
val yarnConf = new YarnConfiguration()
val yarnClient = YarnClient.createYarnClient()
yarnClient.init(yarnConf)
yarnClient.start()
try {
import scala.collection.JavaConversions._
//YarnApplicationState有应用不同的运行状态,如RUNNING、FINISHED
val applicationReports = yarnClient.getApplications(util.EnumSet.of(YarnApplicationState.RUNNING))
applicationReports.foreach { ar =>
//应用ID,spark.app.id
println(s"appId ==> ${ar.getApplicationId}")
//应用名字,spark.app.name
println(s"name ==> ${ar.getName}")
//应用所处队列,spark.yarn.queue
println(s"queue ==> ${ar.getQueue}")
//应用的用户
println(s"user ==> ${ar.getUser}")
}
} catch {
case e: Exception => e.printStackTrace()
}
yarnClient.stop()
}