ds job调度脚本

最新推荐文章于 2024-07-16 08:45:00 发布

pianzif

最新推荐文章于 2024-07-16 08:45:00 发布

阅读量2.9k

点赞数

分类专栏： datastage 文章标签： ds job 脚本

本文链接：https://blog.csdn.net/fuwencaho/article/details/27970339

版权

datastage 专栏收录该内容

20 篇文章 0 订阅

订阅专栏

在自己工作中需要调度job，在自带的director感觉不是很方便，一下是收集的几种调度脚本！

脚本一：

/test/runJob.sh

#!/bin/bash

########################################

#

# runJob.sh 2012-08-19

# run a job with parameters

#

#######################################

# if the number of input parameters is less than 2,then output the help document and exit

if [ $# -lt 2 ] ; then

cat << HELP

runJob --run a job UASGE: runJob projectName jobName jobParameters

EXAMPLE: runJob dsstage1 DD_Test -param startDT=20120819 -param endDT=20120819

HELP

exit 0

fi

projectName="$1"

jobName="$2"

jobParameters="$3"

#echo $projectName

#echo $jobName

#echo $jobParameters

logdir=/DS/DSLogs #directory to store logs

workdate=`date +%Y%m%d`

#logdir processing.If log folder not exists,create folder.

if [ -d $logdir ]; then

echo "$logdir is exist,continue..."

else

echo "$logdir is not exist,creating $logdir..."

mkdir -p $logdir

fi

#job state processing.If job state is not finished ok,then reset the job

jobsta=$($DSHOME/bin/dsjob -jobinfo $projectName $jobName 2>&1 | awk -F: '/^Job Status/{print $2}')

echo $jobsta

if [ "$jobsta" == " RUN FAILED (3)" -o "$jobsta" == " STOPPED (97)" ];then

echo "Reset before run job $jobname"

$DSHOME/bin/dsjob -run -mode RESET $projectName $jobName >>${logdir}/job_init_` date +%Y%m%d`.log

sleep 5

fi

#run a job

dsjob -run -mode NORMAL $jobparameters -jobstatus $projectName $jobName >> /DS/DSLogs/job_run_` date +%Y%m%d`.log

来自：http://www.cnblogs.com/BlueBreeze/archive/2012/11/02/2751067.html

脚本二:

#!/bin/ksh 
####### 
################################################################################ 
####### 
####### FILE: MasterControl.ksh 
####### 
####### DESCRIPTION: Starts a DataStage MasterControl type job passing 
#######              all runtime parameter values 
####### 
####### 
####### Date       Version   Developer      Description 
####### ---------- --------- -------------- ------------------------------------ 
####### 2002-05-15 1.0       Ken Bland      Initial release 
####### 2002-06-27 2.2       Ken Bland      FileSetDate/ProcessDate modifications 
####### 2002-07-10 2.2       Steve Boyce    Added ProcessDate as 14th parameter 
####### 2002-08-16 2.3       Steve Boyce    Now calls stored procedure 
#######                                     GET_NEW_BATCH_NBR in datastage schema 
#######                                     instead of deriving it and inserting 
#######                                     here. 
#######                                     Uses comSQLPlus.ksh and comPLSQL.ksh 
#######                                     instead of SQLPlusStub.ksh.ksh 
####### 

PROG=`basename ${0}` 
EXIT_STATUS=0 

NOW=`date` 
echo "${NOW} ${PROG} Initialization..." 
echo 

####### 
####### CONFIGURATION ########################################################## 
####### 

if [ ${#} -ne 14 ]; then 
   echo "${NOW} ${PROG} : Invalid parameter list." 
   echo "${NOW} ${PROG} : The script needs 14 parameters:" 
   echo "${NOW} ${PROG} :    JobName" 
   echo "${NOW} ${PROG} :    ParameterFile" 
   echo "${NOW} ${PROG} :    FileSetDate (YYYY-MM-DD)" 
   echo "${NOW} ${PROG} :    BatchNumber" 
   echo "${NOW} ${PROG} :    JobHierarchyFile" 
   echo "${NOW} ${PROG} :    SourceSystemList" 
   echo "${NOW} ${PROG} :    SubjectAreaList" 
   echo "${NOW} ${PROG} :    ClearWorkArea" 
   echo "${NOW} ${PROG} :    StartingMilestone" 
   echo "${NOW} ${PROG} :    EndingMilestone" 
   echo "${NOW} ${PROG} :    DebugMode" 
   echo "${NOW} ${PROG} :    JobLinkStatisticChecksFile" 
   echo "${NOW} ${PROG} :    ResurrectLogFile" 
   echo "${NOW} ${PROG} :    ProcessDate (NULL|YYYY-MM-DD H24:MI:SS)" 
   exit 99 
fi 

JobName="${1}" 
ParameterFile="${2}" 
FileSetDate="${3}" 
BatchNumber="${4}" 
JobHierarchyFile="${5}" 
SourceSystemList="${6}" 
SubjectAreaList="${7}" 
ClearWorkArea="${8}" 
StartingMilestone="${9}" 
EndingMilestone="${10}" 
DebugMode="${11}" 
JobLinkStatisticChecksFile="${12}" 
ResurrectLogFile="${13}" 
ProcessDate="${14}" 

echo "${NOW} ${PROG} JobName ${JobName}" 
echo "${NOW} ${PROG} ParameterFile ${ParameterFile}" 
echo "${NOW} ${PROG} FileSetDate ${FileSetDate}" 
echo "${NOW} ${PROG} BatchNumber ${BatchNumber}" 
echo "${NOW} ${PROG} JobHierarchyFile ${JobHierarchyFile}" 
echo "${NOW} ${PROG} SourceSystemList ${SourceSystemList}" 
echo "${NOW} ${PROG} SubjectAreaList ${SubjectAreaList}" 
echo "${NOW} ${PROG} ClearWorkArea ${ClearWorkArea}" 
echo "${NOW} ${PROG} StartingMilestone ${StartingMilestone}" 
echo "${NOW} ${PROG} EndingMilestone ${EndingMilestone}" 
echo "${NOW} ${PROG} DebugMode ${DebugMode}" 
echo "${NOW} ${PROG} JobLinkStatisticChecksFile ${JobLinkStatisticChecksFile}" 
echo "${NOW} ${PROG} ResurrectLogFile ${ResurrectLogFile}" 
echo "${NOW} ${PROG} ProcessDate ${ProcessDate}" 
echo 

# Below will look in the parameters.ini file to determine the directory path each. 
UserID=`whoami` 
BinFileDirectory=`cat /.dshome`/bin 
LogFileDirectory=`grep -w LogFileDirectory ${ParameterFile}|cut -d "=" -f2` 
TempFileDirectory=`grep -w TempFileDirectory ${ParameterFile}|cut -d "=" -f2` 
CommonScriptFileDirectory=`grep -w CommonScriptFileDirectory ${ParameterFile}|cut -d "=" -f2` 
CommonLogFileDirectory=`grep -w CommonLogFileDirectory ${ParameterFile}|cut -d "=" -f2` 
LogFileName=${CommonLogFileDirectory}/${PROG}_${JobName}.log 
TEMPBATCHNBRLOG=${TempFileDirectory}/${PROG}_${JobName}_start.log 
DATASTAGEPROJECT=`grep -w DATASTAGEPROJECT ${ParameterFile}|cut -d "=" -f2` 
DSSERVER=`grep -w DSSERVER ${ParameterFile}|cut -d "=" -f2` 
DSUSERID=`grep -w DSUSERID ${ParameterFile}|cut -d "=" -f2` 
DSPASSWORD=`grep -w DSPASSWORD ${ParameterFile}|cut -d "=" -f2` 

NOW=`date` 
echo "${NOW} ${PROG} UserID ${UserID}" 
echo "${NOW} ${PROG} BinFileDirectory ${BinFileDirectory}" 
echo "${NOW} ${PROG} LogFileDirectory ${LogFileDirectory}" 
echo "${NOW} ${PROG} TempFileDirectory ${TempFileDirectory}" 
echo "${NOW} ${PROG} CommonScriptFileDirectory ${CommonScriptFileDirectory}" 
echo "${NOW} ${PROG} CommonLogFileDirectory ${CommonLogFileDirectory}" 
echo "${NOW} ${PROG} LogFileName ${LogFileName}" 
echo "${NOW} ${PROG} TEMPBATCHNBRLOG ${TEMPBATCHNBRLOG}" 
echo "${NOW} ${PROG} DATASTAGEPROJECT ${DATASTAGEPROJECT}" 
echo "${NOW} ${PROG} DSSERVER ${DSSERVER}" 
echo "${NOW} ${PROG} DSUSERID ${DSUSERID}" 
echo "${NOW} ${PROG} DSPASSWORD *Protected*" 
echo 

####### 
####### PARAMETER BUILD Without batch number ################################## 
####### 

if [ "${ProcessDate}" = "NULL" ]; then 
   StartTimestamp=`date '+%Y-%m-%d %H:%M:%S'` 
else 
   StartTimestamp="${ProcessDate}" 
fi 
ParamList=" -param ParameterFile=${ParameterFile}" 
ParamList="${ParamList} -param ProcessDate=\"${StartTimestamp}\"" 
ParamList="${ParamList} -param FileSetDate=${FileSetDate}" 
ParamList="${ParamList} -param JobHierarchyFile=${JobHierarchyFile}" 
ParamList="${ParamList} -param SourceSystemList=${SourceSystemList}" 
ParamList="${ParamList} -param SubjectAreaList=${SubjectAreaList}" 
ParamList="${ParamList} -param ClearWorkArea=${ClearWorkArea}" 
ParamList="${ParamList} -param StartingMilestone=${StartingMilestone}" 
ParamList="${ParamList} -param EndingMilestone=${EndingMilestone}" 
ParamList="${ParamList} -param DebugMode=${DebugMode}" 
ParamList="${ParamList} -param JobLinkStatisticChecksFile=${JobLinkStatisticChecksFile}" 
ParamList="${ParamList} -param ResurrectLogFile=${ResurrectLogFile}" 

####### 
####### Get Batch Number and create ETL_BATCH_AUDIT record ##################### 
####### 

echo "${NOW} ${PROG} About to get new BATCH_NBR and insert it into ETL_BATCH_AUDIT..." 
${CommonScriptFileDirectory}/comPLSQL.ksh ${ParameterFile} "IRDSN" "IRUserID" \ 
                                           GET_NEW_BATCH_NBR \ 
                                              "${JobName}" \ 
                                              "${StartTimestamp}" \ 
                                              "${UserID}" \ 
                                              "${SourceSystemList}" \ 
                                              "${SubjectAreaList}" \ 
                                              "${ParamList}" \ 
                                              "${FileSetDate}" > ${TEMPBATCHNBRLOG} 
SQL_EXIT_STATUS=$? 
cat ${TEMPBATCHNBRLOG} 
if [ "${SQL_EXIT_STATUS}" != 0 ]; then 
   NOW=`date` 
   echo "${NOW} ${PROG} Failure to connect/insert into ETL_Batch_Audit table!" 
   exit ${SQL_EXIT_STATUS} 
fi 

####### 
####### Get BATCH_NBR from batch number log file ############################## 
####### 
BatchNumber=`grep -w BATCH_NBR ${TEMPBATCHNBRLOG}|cut -d "=" -f2` 
if [ -z "${BatchNumber}" ]; then 
   NOW=`date` 
   echo "${NOW} ${PROG} Failure to retrieve BATCH_NBR from ${TEMPBATCHNBRLOG}" 
   exit ${SQL_EXIT_STATUS} 
fi 

####### 
####### Add batch number to list of parameters ################################# 
####### 
ParamList="${ParamList} -param BatchNumber=${BatchNumber}" 

NOW=`date` 
echo 
echo ${NOW} ${PROG} Parameter list: ${ParamList} 
echo 

####### 
####### DataStage EXECUTION #################################################### 
####### 

NOW=`date` 
echo "${NOW} ${PROG} Executing DataStage dsjob program..." 

echo ${BinFileDirectory}/dsjob -server ${DSSERVER} -user ${DSUSERID} -password ${DSPASSWORD} -run -wait ${ParamList} ${DATASTAGEPROJECT} ${JobName} 2>&1 > ${LogFileName} 
echo 
echo "${BinFileDirectory}/dsjob -server ${DSSERVER} -user ${DSUSERID} -password ${DSPASSWORD} -run -wait ${ParamList} ${DATASTAGEPROJECT} ${JobName} 2>&1 > ${LogFileName}" 
echo 

eval ${BinFileDirectory}/dsjob -server ${DSSERVER} -user ${DSUSERID} -password ${DSPASSWORD} -run -wait ${ParamList} ${DATASTAGEPROJECT} ${JobName} 2>&1 >> ${LogFileName} 

jobwaiting=`grep "Waiting for job..." ${LogFileName}` 
if [ "${jobwaiting}" != "Waiting for job..." ]; then 
   NOW=`date` 
   echo ${NOW} ${PROG} "DataStage failed to start the job" 
   failedstart=1 
else 
   NOW=`date` 
   echo ${NOW} ${PROG} "DataStage successfully started the job" 
   failedstart=0 
fi 
NOW=`date` 
echo ${NOW} ${PROG} "Retrieving job information" 

${BinFileDirectory}/dsjob -server ${DSSERVER} -user ${DSUSERID} -password ${DSPASSWORD} -jobinfo ${DATASTAGEPROJECT} ${JobName} >> ${LogFileName} 

####### 
####### CHECK STATUS ########################################################### 
####### 

ERROR=`grep "Job Status" ${LogFileName}` 
ERROR=${ERROR##*\(} 
ERROR=${ERROR%%\)*} 

if [ "${failedstart}" != 0 ]; then 
   NOW=`date` 
   echo ${NOW} ${PROG} "The job failed to start" 
   AuditStatus="FAILURE" 
   Comments="MasterControl aborted" 
   EXIT_STATUS=1 
else 
   if [ "${ERROR}" = 1 -o "${ERROR}" = 2 ]; then 
      NOW=`date` 
      echo ${NOW} ${PROG} "The job completed successfully" 
      AuditStatus="SUCCESS" 
      Comments="" 
      EXIT_STATUS=0 
   else 
      NOW=`date` 
      echo ${NOW} ${PROG} "The job aborted" 
      AuditStatus="FAILURE" 
      Comments="MasterControl aborted" 
      EXIT_STATUS=1 
   fi 
fi 

FailedJobCount=`grep -i FAILED ${LogFileDirectory}/${JobName}.log|wc -l|cut -b1-9` 
FailedJobCount=`expr ${FailedJobCount} + 0` 
echo ${NOW} ${PROG} The number of failed jobs is [${FailedJobCount}] 

if [ "${FailedJobCount}" != 0 ]; then 
   NOW=`date` 
   echo ${NOW} ${PROG} "The job had failed processes" 
   AuditStatus="FAILURE" 
   Comments="MasterControl had ${FailedJobCount} failed processes" 
   EXIT_STATUS=1 
fi 
StoppedJobStreamCount=`grep "JOB STREAM STOPPED" ${LogFileDirectory}/${JobName}.his|wc -l|cut -b1-9` 
StoppedJobStreamCount=`expr ${StoppedJobStreamCount} + 0` 
if [ "${StoppedJobStreamCount}" != 0 ]; then 
   NOW=`date` 
   echo ${NOW} ${PROG} "The job stream was STOPped or KILLed" 
   AuditStatus="FAILURE" 
   Comments="MasterControl job stream was STOPped or KILLed" 
   EXIT_STATUS=1 
fi 

####### 
####### AUDIT ################################################################## 
####### 

echo 
echo "${NOW} ${PROG} About to update ETL_BATCH_AUDIT with status information..." 
EndTimestamp=`date '+%Y-%m-%d %H:%M:%S'` 

SQLstring="UPDATE ETL_BATCH_AUDIT A \ 
              SET A.END_TIMESTAMP = TO_DATE('${EndTimestamp}','YYYY-MM-DD HH24:MI:SS'), \ 
                         A.STATUS = '${AuditStatus}', \ 
                       A.COMMENTS = '${Comments}', \ 
                A.RUNTIMESETTINGS = '${ParamList}' \ 
            WHERE (A.BATCH_NBR = ${BatchNumber});" 

NOW=`date` 
echo ${NOW} ${PROG} Audit SQL ${SQLstring} 
SQLScriptFileName=${TempFileDirectory}/${PROG}_${JobName}_end.sql 
echo ${SQLstring} > ${SQLScriptFileName} 

${CommonScriptFileDirectory}/comSQLPlus.ksh ${ParameterFile} IRDSN IRUserID ${SQLScriptFileName} 
SQL_EXIT_STATUS=$? 
if [ "${SQL_EXIT_STATUS}" != 0 ]; then 
   NOW=`date` 
   echo ${NOW} ${PROG} Failure to connect/update into ETL_Batch_Audit table! 
   exit ${SQL_EXIT_STATUS} 
fi 

####### 
####### EXIT ################################################################### 
####### 

NOW=`date` 
echo ${NOW} ${PROG} Complete, exiting with status [${EXIT_STATUS}] 
exit ${EXIT_STATUS}

来自：http://www.blogjava.net/xichengmylove/archive/2007/07/18/130982.html

脚本三：

#!/bin/bash
. /home/dsadm/.bash_profile
CRMHOME=/home/dsadm/crm

##一些获取时间函数


get_accdate()
{
FF=/home/dsadm/crm/get_accdate.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "select accdate from ifs_bankctl ; " >> $FF
echo "disconnect current;" >>$FF
ACCDATE=`db2 -tf $FF | grep "20[0-9][0-9]-[0-1][0-9]-" `
return 0
}
get_mend()
{
FF=/home/dsadm/crm/get_mend.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "select 'AA='||substr(char(nextaccdate),9,2) from ifs_bankctl ; " >> $FF
echo "disconnect current;" >>$FF
MEND=`db2 -tf $FF | grep "AA"|cut -d= -f2 `
return 0
}
get_qend()
{
FF=/home/dsadm/crm/get_qend.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "select 'AA='||substr(replace(char(nextaccdate),'-',''),5,4) from ifs_bankctl ; " >> $FF
echo "disconnect current;" >>$FF
QEND=`db2 -tf $FF | grep "AA"|cut -d= -f2 `
return 0
}
get_date()
{
FF=/home/dsadm/crm/get_date.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "select replace(char(accdate),'-','') from ifs_bankctl ; " >>$FF
echo "disconnect current;" >>$FF
DATE=`db2 -tf $FF | grep "20[0-9][0-9][0-1][0-9]" `
}
#

get_weekdate())
{
FF=/home/dsadm/crm/get_week.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "select  'WEEKDATE='||CHAR(DAYOFWEEK(accdate)) from ifs_bankctl; " >> $FF
echo "disconnect current;" >>$FF
WEEKDATE=`db2 -tf $FF | grep "WEEKDATE" |cut -d= -f2 `
return 0
}
ifs_jobstart()
{
pro_tab=$1
g_accdate=$2
FF=/home/dsadm/crm/ifs_jobstart.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "update ifs_joblog set startdate=current timestamp,sqlcode='6',enddate=null where proname='$pro_tab' and accdate=DATE('$g_accdate'); " >> $FF
echo "disconnect current;" >>$FF
db2 -tf $FF 
}
ifs_jobend()
{
pro_tab=$1
g_sqlcode=$2
FF=/home/dsadm/crm/ifs_jobend.sql
echo "connect to crmdb user crmuser using crmuser;" >$FF
echo "update ifs_joblog set sqlcode='$g_sqlcode',enddate=current timestamp where proname='$pro_tab' and startdate is not null; " >> $FF
echo "disconnect current;" >>$FF
db2 -tf $FF 
}
TABLELIST=/home/dsadm/crm/dsjob_list.conf
DSHOME=/opt/IBM/InformationServer/Server/DSEngine
cd $DSHOME
. ./dsenv
>$CRMHOME/dsjob_status.dat
jobstatus="0"


#####
##作业开始执行
#####


for i in `cat $TABLELIST`                     # -----作业配置文件             
do
DSJOB=`echo $i|cut -d: -f1`                   #  ----取作业名
DSM=`echo $i|cut -d: -f2`					#	----取作业执行频率
get_accdate
get_date
cd $CRMHOME                       
if [ -d $DATE ]; then
 echo ""
else
 mkdir $DATE
fi
if [ "$jobstatus" = 2 ]            # ------1为正常 ，2为错误  ？？？？？？？？？？？
   then 
       echo $DSJOB:$jobstatus:$DSM>>dsjob_status.dat
else
cd $DATE
ifs_jobstart $DSJOB $ACCDATE
get_mend
echo $MEND
get_qend
echo $QEND
get_weekdate
echo $WEEKDATE
MENDFLAG=01
QENDFLAG1=0101
QENDFLAG2=0401
QENDFLAG3=0701
QENDFLAG4=1001
WEEKFLAG=1
if [ "$DSM" = "M" -a  "$MEND" != "$MENDFLAG" -o "$DSM" = "Q" -a  "$QEND" != "$QENDFLAG1" -a  "$QEND" != "$QENDFLAG2" -a  "$QEND" != "$QENDFLAG3" -a  "$QEND" != "$QENDFLAG4"  -o "$DSM" = "Y" -a
  "$QEND" != "$QENDFLAG1"  -o "$DSM" = "W" -a  "$WEEKDATE" != "$WEEKFLAG" ] 
   then jobstatus="1"
   echo "today is not monthend or quterend or yearend pass"
ifs_jobend $DSJOB $jobstatus
else
$DSHOME/bin/dsjob -run -mode RESET -warn 0 -jobstatus lscrm $DSJOB>$DSJOB.log
$DSHOME/bin/dsjob -run -mode NORMAL -warn 0 -jobstatus lscrm $DSJOB>>$DSJOB.log


####执行之后检查日志

#成功

runok=`cat $DSJOB.log|grep 1`
    if [ -n "$runok" ]
    then
        echo "finish ok"
        jobstatus="1"
    fi
echo 'fg='$runok
echo $jobstatus

#有警告
runwarn=`cat $DSJOB.log|grep 2`
    if [ -n "$runwarn" ]
    then
        echo "finish with warnning"
        jobstatus="1"
    fi
echo 'xx='$runwarn
echo $jobstatus

#有错误
runerror=`cat $DSJOB.log|grep 3`
    if [ -n "$runerror" ]
    then
        echo "finish with error"
        jobstatus="2"
    fi
echo 'ee='$runerror
echo $jobstatus   
echo  $DSJOB

#写入作业运行状态文件
echo $DSJOB:$jobstatus:$DSM>>$CRMHOME/dsjob_status.dat

#更行数据库状态日志
ifs_jobend $DSJOB $jobstatus
fi
fi
done
exit 0

来自我的！其实有很多地方需要改动，但是不是我说能改就能改的！