大数据任务脚本

最新推荐文章于 2022-05-30 09:18:07 发布

小帅热爱难回头

最新推荐文章于 2022-05-30 09:18:07 发布

阅读量233

点赞数

分类专栏： Shell脚本

本文链接：https://blog.csdn.net/weixin_44695980/article/details/93847767

版权

Shell脚本专栏收录该内容

2 篇文章 0 订阅

订阅专栏

#!/bin/sh

set -o errexit

bindir=$(cd $(dirname $0); p w d) e c h o " h o m e =$ bindir"

day=date -d -1hour +%Y-%m-%d
hour=date -d -1hour +%H

if [ $# -eq 2 ];then
echo “hello~~~~~”
day=$1
hour=$2
fi

echo "day:" $d a y e c h o " h o u r : "$ hour

echo “====rm local jar”
rm -rf $bindir/*.jar

res=$?
if [ $res -ne 0 ];then
exit $?
fi

#rm -rf spark-etl.jar
echo “===down jars file: /footstone/hbase/wifi/jars”
hadoop fs -get /footstone/hbase/wifi/jars/test-project-1.0-SNAPSHOT.jar
res=$?
if [ $res -ne 0 ];then
exit $res
fi

/opt/cloudera/parcels/CDH-6.0.1-1.cdh6.0.1.p0.590678/bin/spark-submit
–class bigdata.Work
–master yarn \
–deploy-mode client
–conf “spark.yarn.executor.memoryOverhead=500”
–conf “spark.ui.enabled=false”
–conf “spark.executor.heartbeatInterval=30s”
–conf “spark.default.parallelism=10” \
–conf “spark.storage.memoryFraction=0.4”
–executor-memory 3g --executor-cores 1 --num-executors 60
–driver-memory 2g
–driver-class-path “$bindir/bigdata.properties” \
$bindir/test-project-1.0-SNAPSHOT.jar $day $hour

res=$?
if [ $res -ne 0 ];then
exit $res
fi