crontab 例行任务汇总及实践

1. crontab 任务说明

执行crontab -e 命令可进入定时任务设置页面，编辑方法同vim。

# For example, you can run a backup of all your user accounts
# at 5 a.m every week with:
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/

1.1 设置定时任务

# 每早八点，以后台任务的形式执行test.sh脚本
00 08 * * * cd /home/zz/work_dir && nohup bash -x test.sh > logs/log_$(date +'\%Y-\%m-\%d') 2>&1 &

# 每5min执行一次
*/5 * * * * bash -x test.sh 

# 每周五下午4:30执行一次
30 16 * * 5 bash -x test.sh 

注意：此处需要在work_dir目录手动创建logs文件夹，否则会找不到日志。

1.2 查看crontab内容

crontab -l

1.3 探索与实践

Demo1:

#!/bin/bash

if [ $# = 0 ]; then
    current_dt=$(date "+%Y-%m-%d")
elif [ $# = 1 ]; then
    current_dt=$1
else
      echo "too many parmars!"
      exit -1
fi

echo ${current_dt}

last_dt=`date -d "1 day ago $current_dt" +%Y-%m-%d`
# last_dt=$(date --date='2 day ago' +%Y-%m-%d)
# last_dt=`date -d $current_dt'-1 day' +%F`
last_2dt=`date -d "2 day ago $current_dt" +%Y-%m-%d`
last_7dt=`date -d "7 day ago $current_dt" +%Y-%m-%d`
last_30dt=`date -d "30 day ago $current_dt" +%Y-%m-%d`
echo ${last_dt}
echo ${last_2dt}
echo ${last_7dt}
echo ${last_30dt}

/app/hadoop/hive/bin/hive -e "
insert overwrite table table_name partition(pt='${last_dt}')
select id 
,min(fupload_dt) as fupload_dt --首次上传日期
,max(lupload_dt) as lupload_dt --最近一次上传日期
from(
    select id
    ,fupload_dt
    ,lupload_dt
    ,pt
    from table_name
    where dt='${last_2dt}'

    union all

    select id
    ,dt as fupload_dt
    ,dt as lupload_dt
    ,pt
    from XX 
    where dt='${last_dt}'
) t
group by id
;

#只保留7天数据
alter table table_name drop partition(pt<'${last_7dt}');
"
echo "---insert table_name done!"

demo2 :

:<<eof 
#创建时间：2020-08-15 
#输入数据源： 
#输出结果表： 
#报表中文名称:XX数据 
#报表输出结果文件： 
#开发者： 
#版本号：1.0 
#更新时间： 
#更新内容：  
eof

kinit -kt /home/zz/zz.keytab zz@CENTER.DATAPLAT

if [ $# = 1 ]; then  
    start_dt=$1 
    end_dt=$1 
elif [ $# = 2 ]; then 
    start_dt=$1 
    end_dt=$2 
elif [ $# = 0 ]; then 
    start_dt=`date -d '-1 day' +%F` 
    end_dt=`date -d '-1 day' +%F` 
fi 

start_sec=`date -d $start_dt +%s` 
end_sec=`date -d $end_dt +%s` 

for ((i=$start_sec;i<=$end_sec;i+=86400)) 
do 
    k=`date -d @$i +%F`   
    echo $k 
    /app/hadoop/hive/bin/hive -e " 
        use zz;
        insert overwrite table XX_d_incr partition(pt) 
        select * 
        from table_name;
    " 
    if [ $? != 0 ]; then 
        echo '-------fail------'
        exit 1 
    fi 
done