使用adaptive版本:
spark-submit \
--master yarn \
--deploy-mode client \
--num-executors 6 \
--executor-memory 24G \
--executor-cores 4 \
--driver-memory 4G \
--conf spark.driver.allowMultipleContexts=false \
--conf spark.sql.adaptive.enabled=true \
--conf spark.sql.skewJoin.enabled=true \
--conf spark.sql.adaptive.coalescePartitions.enabled=false \
--conf spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes=128m \
--conf spark.sql.adaptive.skewJoin.skewedPartitionFactor=5 \
--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=64m \
--conf spark.sql.shuffle.partitions=20000 \
关闭adaptive版本:
spark-submit \
--master yarn \
--deploy-mode client \
--num-executors 6 \
--executor-memory 20G \
--executor-cores 4 \
--driver-memory 1G \
--conf spark.driver.allowMultipleContexts=false \
--conf spark.sql.adaptive.enabled=false \
--conf spark.sql.shuffle.partitions=20000 \