#!/bin/bash
# 检查是否有每日例行任务
function wait_job_done(){
pid=`ps -ef|grep job_name |grep -v grep |awk '{print $2}'|wc -l`
until [[ $pid -eq 0 ]];do
this_date=`date +"%y-%m-%d %H:%M:%S"`
echo "job is runing:"$this_date
pid=`ps -ef|grep job_name|grep -v grep |awk '{print $2}'|wc -l`
sleep 30
done
}
# 几个不同时间格式化
# date -d @1584413171 +"%Y-%m-%d %H:00:00"
# this_hour=`date +"%y-%m-%d %H:00:00"`
# last_hour=`date +"%y-%m-%d %H:00:00" -d "-1hours"` minutes seconds days months years
# 传入不同时间间隔的时间戳 $1 为不同小时,比如1 小时
function git_diff_hours_timestamp(){
for i in {10..1032..1};do
j=$(($i+1))
h1=$((1584413171-3600*$i*$1))
h2=$((1584413171-3600*$j*$1))
echo $h2","$h1
done
}
# 提交 spark 任务 ,其中时间格式要用加上双引号
function run_job(){
job_name=`date -d @$3 +"%Y-%m-%d->%H"`
st_time=`date +%s`
spark-submit --class com.upneo.Update \
--master yarn \
--name "update_ak"$job_name \
--executor-memory 4g \
--num-executors 5 \
--executor-cores 1 \
--jars target/update-1.0-SNAPSHOT.jar target/update-1.0-SNAPSHOT-jar-with-dependencies.jar "$1" "$2" 100
wait
end_time=`date +%s`
use_time=`echo "scale=2;("$end_time-$st_time")/60"|bc`
curl 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=0' \
-H 'Content-Type: application/json' \
-d '{"msgtype": "text","text": {"content": "'before_neo4j":"$job_name"点,更新完毕,用时:"$use_time"分钟"'"}}'
}
for i in `cat date_job.txt`;do
# 按照符号切割
arr=(${i//,/ })
st_time=`date -d @${arr[0]} +"%Y-%m-%d %H:00:00"`
ed_time=`date -d @${arr[1]} +"%Y-%m-%d %H:00:00"`
# 检查是否有正在运行的任务
wait_job_done
echo $st_time"========="$ed_time
run_job "$st_time" "$ed_time" "${arr[1]}"
done