#!/bin/bash
# chmod +x ./test.sh 修改权限
num=0
while(($num<2181))
do
spark-submit \
--master yarn \
--deploy-mode cluster \
--queue default \
--num-executors 45 \
--executor-memory 2G \
--driver-memory 24G \
--archives \
--py-files , \
--conf spark.dynamicAllocation.enabled=false \
--conf spark.yarn.maxAppAttempts=1 \
--conf spark.yarn.executor.memoryOverhead=12288 \
myfloder.py \
--num $num \
--output predictions
#num=$[num+6]
#echo $num
let "num+=6" # 自加6
done
shell执行spark-submit
最新推荐文章于 2024-01-03 17:22:41 发布