spark参数设置

//設置sparkconf參數
val sparkConf = new SparkConf() //.setAppName("DirectKafka").set("spark.task.maxFailures", "1")
sparkConf.set("spark.rpc.askTimeout", "1200") //设置20分钟
//spark.network.timeout
sparkConf.set("spark.network.timeout", "1200")
//spark.core.connection.ack.wait.timeout
sparkConf.set("spark.core.connection.ack.wait.timeout", "1200")
sparkConf.set("spark.core.connection.auth.wait.timeout", "1200")
//spark.yarn.max.executor.failures
sparkConf.set("spark.yarn.max.executor.failures", "300")
//spark.akka.timeout
sparkConf.set("spark.akka.timeout", "1200")
//spark.rpc.lookupTimeout
sparkConf.set("spark.rpc.lookupTimeout", "1200")

//confs.set("spark.driver.memory", "5g")
sparkConf.set("spark.speculation", "true")
sparkConf.set("spark.shuffle.consolidateFiles", "true")

sparkConf.set("spark.rdd.compress", "true")
sparkConf.set("spark.storage.memoryFraction", "1")
sparkConf.set("spark.core.connection.ack.wait.timeout", "6000")
sparkConf.set("spark.akka.frameSize", "50")


sparkConf.set("dfs.client.slow.io.warning.threshold.ms", "100000")
sparkConf.set("yarn.resourcemanager.connect.max-wait.ms", Integer.MAX_VALUE.toString())
sparkConf.set("mapred.task.timeout", "1800000")
sparkConf.set("dfs.socket.timeout", "6000000")