Spark 配置调优

  spark.master                     spark://192.168.1.10:7077
  spark.eventLog.enabled           true
  spark.eventLog.dir               hdfs://192.168.1.9:9000/sparkEventLog
  spark.serializer                 org.apache.spark.serializer.KryoSerializer
  spark.driver.memory              6g

spark.executor.extraJavaOptions -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent
  spark.executor.memory            8g
  spark.executor.userClassPathFirst   false
  spark.streaming.stopGracefullyOnShutdown true
  spark.kryoserializer.buffer.max 1536m
  spark.default.parallelism 16
# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
  spark.scheduler.allocation.file  /opt/spark-2.1.0-bin-hadoop2.7/conf/fairscheduler.xml
  spark.scheduler.pool   production
  spark.event.listener.logRate  5000
  spark.event.listener.logEnable true
  spark.eventQueue.size 10000000
  spark.network.timeout 800
  spark.executor.overhead.memory 2g
  spark.locality.wait.node 8
  spark.locality.wait.rack 5
  spark.shuffle.sort.bypassMergeThreshold 600
  spark.shuffle.io.retryWait 30
  spark.shuffle.io.maxRetries 10
  spark.scheduler.listenerbus.eventqueue.capacity 10000000