Spark常用提交脚本

1. 运行脚本：

#!/usr/bin/env bash

input="XXXX" 
output="XXXX"
sday="XXXX"
id_type="XXXX"
tag_origin_short_name="XXXX"

name=dataToES_test

if [ ! -d "/opt/log/wangxuan/norman/${sday}/${id_type}/${tag_origin_short_name}/sparkESLog" ]; then
  mkdir -p /opt/log/wangxuan/norman/${sday}/${id_type}/${tag_origin_short_name}/sparkESLog
fi

# 预清空输出的hdfs路径
hdfs dfs -rm -r XXXX

time1=$(date "+%Y-%m-%d %H:%M:%S")
echo "------------------${time1} ${sday} ${id_type} ${tag_origin_short_name} spark任务启动------------------"

set -x
#新脚本需要注意修改当前服务器的spark bin路径
/usr/lib/software/spark/spark-2.3/bin/spark-submit --class XXXX \
    --master yarn \
    --deploy-mode cluster \
    --queue root.offline.hdp_teu_dpd.normal \
    --name ${name} \
    --executor-memory 20G \
    --num-executors 64 \
    --executor-cores 4 \
    --conf spark.yarn.executor.memoryOverhead=4G \
    --conf "spark.driver.extraJavaOptions=-Dlog4jspark.root.logger=WARN,console" \
    --conf "spark.executor.extraJavaOptions=-verbose:gc -XX:+UseG1GC" \
    --driver-memory 4g \
    --conf spark.yarn.maxAppAttempts=1 \
    --conf spark.locality.wait=60000 \
    --conf spark.dynamicAllocation.enabled=true \
    --conf spark.rpc.askTimeout=6000 \
    --conf spark.core.connection.ack.wait.timeout=6000 \
    --conf spark.storage.memoryFraction=0.5 \
    --conf spark.shuffle.memoryFraction=0.3 \
    --conf spark.rpc.netty.dispatcher.numThreads=64 \
    --conf spark.default.parallelism=3000 \
    ./XXXX.jar ${input} ${output} > /opt/log/XXXX/sparkESLog/${name}.log 2>&1

if [ "$?" != 0 ] ; then
   time2=$(date "+%Y-%m-%d %H:%M:%S")
   echo "！！！!${time2} ${sday} ${id_type} ${tag_origin_short_name} spark任务程序强行终止！！！!"
   exit 1
fi

time3=$(date "+%Y-%m-%d %H:%M:%S")
echo "------------------${time3} ${sday} ${id_type} ${tag_origin_short_name} spark任务运行成功------------------"

exit 0

2. 调度运行脚本并打印脚本运行日志的启动脚本

#!/bin/bash
BASEDIR=`dirname $0`
cd ${BASEDIR}/../run
BASEDIR=`pwd`
cd ${BASEDIR}

id_type=$1
project=$2
sday_key=$3

if [ ! -d "/opt/log/XXXX/${sday_key}/${id_type}/${project}" ]; then
  mkdir -p /opt/log/XXXX/${sday_key}/${id_type}/${project}
fi

time1=$(date "+%Y-%m-%d %H:%M:%S")
echo "------------------------${time1} spark作业开始运行------------------------"

./XXXX.sh ${id_type} ${project} ${sday_key} > /opt/log/XXXX/${sday_key}/${id_type}/${project}/2.project_dsToEs_online_${id_type}_${project}.log 2>&1

if [ "$?" != 0 ] ; then
   time2=$(date "+%Y-%m-%d %H:%M:%S")
   echo "------------------------${time2} spark作业运行异常------------------------"
   exit 1
fi

time3=$(date "+%Y-%m-%d %H:%M:%S")
echo "------------------------${time3} spark作业运行完毕------------------------"

exit 0