spark2-submit命令汇总

spark2-submit \
--master yarn \
--deploy-mode client  --class cn.org.nifa.udcp.UdcpStart \
--driver-class-path /disknas/udcp/stats/udcp-processing/conf/ \
--num-executors 10 \
--driver-memory 10g \
--executor-memory 10g \
--executor-cores 2 \
--conf spark.debug.maxToStringFields=1000 \
--conf spark.port.maxRetries=100 \
--conf spark.driver.maxResultSize=10g \
--conf spark.default.parallelism=20 \
--conf spark.sql.shuffle.partitions=20 \
--conf spark.executor.memoryOverhead=4G \
--conf spark.driver.memoryOverhead=4G \
--conf spark.storage.blockManagerTimeoutIntervalMs=100000 \
--conf spark.shuffle.file.buffer=64k \
--conf spark.reducer.maxSizeInFlight=96M \
--conf spark.network.timeout=300s \
--conf spark.rpc.askTimeout=300s \
--conf spark.shuffle.io.serverThreads=8 \
--conf spark.shuffle.service.enabled=true \
--conf "spark.driver.extraJavaOptions=-Dlog4j.configuration=log4j.properties  -Dconfig.resource=application.conf" \
/disknas/udcp/stats/udcp-processing/jar/udcp-processing-2.0-jar-with-dependencies.jar

需要参数，具体选择 。