spark初始化BindException问题源码分析

环境：spark1.4

问题：

new SparkContext(conf) 的时候出现报错，而且自己写的try catch都没用：

java.net.BindException: 地址已在使用: Service 'SparkUI' failed after 16 retries!
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.nio.ch.Net.bind0(Native Method)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.nio.ch.Net.bind(Net.java:444)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.nio.ch.Net.bind(Net.java:436)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:214)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.server.Server.doStart(Server.java:293)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:228)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.ui.JettyUtils$$anonfun$2.apply(JettyUtils.scala:238)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.ui.JettyUtils$$anonfun$2.apply(JettyUtils.scala:238)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1991)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1982)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:238)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.ui.WebUI.bind(WebUI.scala:117)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:448)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:448)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at scala.Option.foreach(Option.scala:236)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.SparkContext.<init>(SparkContext.scala:448)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at com.iapp.utils.utils$.initSpark(utils.scala:21)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at com.iapp.exp.hiveToMysql$.hiveToMysql(hiveToMysql.scala:151)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at com.iapp.exp.hiveToMysql$.main(hiveToMysql.scala:34)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at com.iapp.exp.hiveToMysql.main(hiveToMysql.scala)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at java.lang.reflect.Method.invoke(Method.java:606)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:664)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:169)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:192)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:111)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout)      at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2018/12/04 18:10:39 - P_EXP_D_USER_BASE - (stdout) 18:10:39,132 ERROR ~ Failed to bind SparkUI

 

根据提示查看源码org.apache.spark.ui.WebUI.bind(WebUI.scala:117)

def bind() {
  assert(!serverInfo.isDefined, "Attempted to bind %s more than once!".format(className))
  try {
    serverInfo = Some(startJettyServer("0.0.0.0", port, handlers, conf, name))
    logInfo("Started %s at http://%s:%d".format(className, publicHostName, boundPort))
  } catch {
    case e: Exception =>
      logError("Failed to bind %s".format(className), e)
      System.exit(1)
  }
}

发现这里exit了，难怪自己的代码catch不到！

 

再根据报错提示查看下面源码org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1982)

def portMaxRetries(conf: SparkConf): Int = {
  val maxRetries = conf.getOption("spark.port.maxRetries").map(_.toInt)
  if (conf.contains("spark.testing")) {
    // Set a higher number of retries for tests...
    maxRetries.getOrElse(100)
  } else {
    maxRetries.getOrElse(16)
  }
}

这里发现spark.port.maxRetries没设置的时候，默认从4040端口申请到4040+16，就不再往上申请了；

def startServiceOnPort[T](
    startPort: Int,
    startService: Int => (T, Int),
    conf: SparkConf,
    serviceName: String = ""): (T, Int) = {

  require(startPort == 0 || (1024 <= startPort && startPort < 65536),
    "startPort should be between 1024 and 65535 (inclusive), or 0 for a random free port.")

  val serviceString = if (serviceName.isEmpty) "" else s" '$serviceName'"
  val maxRetries = portMaxRetries(conf)
  for (offset <- 0 to maxRetries) {
    // Do not increment port if startPort is 0, which is treated as a special port
    val tryPort = if (startPort == 0) {
      startPort
    } else {
      // If the new port wraps around, do not try a privilege port
      ((startPort + offset - 1024) % (65536 - 1024)) + 1024
    }
    try {
      val (service, port) = startService(tryPort)
      logInfo(s"Successfully started service$serviceString on port $port.")
      return (service, port)
    } catch {
      case e: Exception if isBindCollision(e) =>
        if (offset >= maxRetries) {
          val exceptionMessage =
            s"${e.getMessage}: Service$serviceString failed after $maxRetries retries!"
          val exception = new BindException(exceptionMessage)
          // restore original stack trace
          exception.setStackTrace(e.getStackTrace)
          throw exception
        }
        logWarning(s"Service$serviceString could not bind on port $tryPort. " +
          s"Attempting port ${tryPort + 1}.")
    }
  }
  // Should never happen
  throw new SparkException(s"Failed to start service$serviceString on port $startPort")
}

 

结论：这个BindException是源码中处理了的，我们catch不到，要解决这个，关键还是要处理好申请资源的时候的并行量，因为服务器资源是有限的；

要么设置更大的spark.port.maxRetries；

要么改写源码，这个比较麻烦。