sparksubmit提交任务 java,spark 提交任务（指定log4j日志配置）

#!/bin/bash

export JAVA_HOME=/usr/java/jdk1.8.0_202

export JRE_HOME=${JAVA_HOME}/jre

export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib

export PATH=${JAVA_HOME}/bin:$PATH

#set hadoop env

export HADOOP_HOME=/opt/cloudera/parcels/CDH

export PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH

#提交任务

spark2-submit \

--master yarn \

--num-executors 6 \

--executor-memory 8G \

--executor-cores 2 \

--driver-memory 16G \

--conf spark.default.parallelism=30 \

--conf spark.yarn.executor.memoryOverhead=2048 \

--conf spark.storage.memoryFraction=0.5 \

--conf spark.shuffle.memoryFraction=0.3 \

--conf spark.dynamicAllocation.enable=false \

--conf "spark.driver.extraJavaOptions=-XX:+UseG1GC -Dlog4j.debug=true" \

--conf "spark.driver.extraJavaOptions=-Dlog4j.configuration=file:/xxx/log4j.properties" \

--conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=file:/xxx/log4j.properties" \

--class xx xxx.jar xxx.properties

来源：https://www.cnblogs.com/chinaboyzzy/p/13139611.html