python之实现每日任务调度功能

python之实现每日任务调度功能

由于项目需要，有许多开发的小程序，需要每日定时执行，但是又不想用linux自用的crontab功能，所以试着项目中的批量调度编排模式，手写了python版的任务调度。

目录结构
pycrontab
  + bin --存放task配置的任务程序
  + +exec.py -- 任务启动程序
  + log
  + tasks --任务配置信息、实例化配置信息
  + tools --工具类
  + +logger.py --日志封装类
  + +thread.py-- 线程封装类
  + config
  + +config.py --系统配置信息
  + pid -- 记录执行任务程序的进程pid
  + taskStatus -- 记录执行任务程序的执行状态
  + workDate --记录当前工作日文件
  + nextworkDate -- 记录下一个工作日文件
  + cronMain.py --任务调度程序
  + cronMain.pid -- 任务调度启动的进程pid

初始化文件 task.ini

[TEST1] 
id=1
runtime=21:00:00
filename=/bin/test/test1.py
calltype=python
max_num=1
isexecute=ON

目前实现的功能：
1.配置化支持每日任务的编排、调度、定时实例化；
2.多线程任务调度；
3.兼容p2p3;
4.按照任务的种类进行日志分割；
5.任务执行前，对任务运行情况进行检查并kill；
6.主程序启动检查(仅允许一个主程序启动)；
7.支持多种语言的程序执行调度；
8.配置化的支持任务执行循环频率；

cronMain.py
from config import config
from tools import logger,thread
import os
import datetime
import json
import signal
import time
import sys
from time import sleep
version=sys.version.split(' ')
#p2.p3兼容设置
if version[0][0] == '2':
	import ConfigParser
	import commands
elif version[0][0] == '3:
	import configoarer as ConfigParser
	import subprocess as commands
else:
     sys.exit(1)
waitTime=config.waittime
init_tasks={}
workpath=os.getcwd()
today=str(datetime.date.today())
logger=logger.Logger(__name__,workpath+'/log/main.log').getLogger()
def getToday():
    return str(datetime.date.today())
def get_next_date():
    today= datetime.date.today()
    return str(today + datetime.timedelta(days=1))
def get_next_next_date():
    today= datetime.date.today()
    return str(today + datetime.timedelta(days=2))
def readConfig():
    tasks={}
    readconfig=ConfigParser.ConfigParser()
    readconfig.read(workpath+'/tasks/tasks.ini')
    sections=readconfig.sections()
    for section in sections:
        tasks[section]=readconfig.items(section)
    return tasks
def writeConfig(tasks,next_day):
    logger.info('next_day=[%s]' % next_day)
    try:
        for key in tasks.keys():
            logger.info(tasks[key])
            writeconfig=ConfigParser.ConfigParser()
            writeconfig.add_section(key)
            for v in tasks[key]:
                writeconfig.set(key,v[0],v[1])
            writeconfig.set(key,'rundate',next_day)
            writeconfig.set(key,'enddate',get_next_next_date())
            writeconfig.set(key,'status','init')
            filename=workpath+'/tasks/'+next_day+'_'+key+'_task.ini'
            logger.info("instantiation-->"+filename)
            with open(filename,'w') as f:
                writeconfig.write(f)
    except Exception as e:
        logger.error('error:[%s]' % repr(e))
def next_task_instantiation():
    try:
        next_day=get_next_date()
        tasks=init_tasks
        if not tasks:
            raise OSError
        logger.info('tasks=[%s]' % json.dumps(tasks))
        writeConfig(tasks,next_day)
        with open(workpath+'/nextworkDate','w') as wf:
            wf.write(next_day)
    except OSError:
        logger.error("Can't open config file: tasks.ini")
def getTodayTasks():
    #read init tasks.ini
    tasks=init_tasks
    today=getToday()
    todayTasks={}
    for key in tasks.keys():
        TaskFile=workpath+'/tasks/'+today+'_'+key+'_task.ini'
        if not os.path.exists(TaskFile):
            logger.error("Taskfile=[%s] is not exists."% TaskFile )
            tmp=list(tasks[key])
            tmp.append(('init','tasks.ini'))
            tmp.append(('rundate',today))
            tmp.append(('enddate',get_next_date()))
            tmp.append(('status','init'))
            todayTasks[key]=listTodict(tmp)
        else:
            logger.info("Taskfile=[%s] is exists."% TaskFile)
            readconfig=ConfigParser.ConfigParser()
            readconfig.read(TaskFile)
            sections=readconfig.sections()
            for section in sections:
                tasks[section]=readconfig.items(section)
            tmp=list(tasks[key])
            todayTasks[key]=listTodict(tmp)
    return todayTasks
def listTodict(listdate):
    task_dict={}
    for i in listdate:
        task_dict[i[0]]=i[1]
    return task_dict
def kill(pid):
    import signal
    pname=os.popen('ps -ef| grep %s | grep -v grep' % pid)
    logger.info("kill pid:[%s]" % pname.readlines())
    try:
        ret=os.kill(int(pid),signal.SIGKILL)
        logger.info("killed pid=[%s],ret=[%s]" % (pid ,str(ret)))
        return ret
    except Exception as e:
        logger.error('error:[%s]' % repr(e))
        return 1
def getOldTaskPids():
    TaskPids={}
    fileslist=os.listdir(workpath+'/pid')
    pid=''
    for f in fileslist:
        tmpPath=workpath+'/pid/'+f
        pid= os.popen('cat %s' % tmpPath)
        appCode=f.split('.')[0]
        TaskPids[appCode]=pid.readlines()[0]
    logger.info('TaskPids=[%s]' % json.dumps(TaskPids))
    return TaskPids
def initTasks():
    global init_tasks
    init_tasks=readConfig()
    today=getToday()
    with open(workpath+'/'+config.workDatePath,'w') as f:
        f.write(today)
    with open(workpath+'/'+config.nextworkDatePath,'w') as f:
        f.write(get_next_date()) 
    logger.info("initTasks=[%s]" % json.dumps(init_tasks))
    todayTasks=getTodayTasks()
    logger.info("todayTask=[%s]" % json.dumps(todayTasks))
    oldTaskPids=getOldTaskPids()
    logger.info('oldTaskPids=[%s]' % json.dumps(oldTaskPids))
    return init_tasks,todayTasks,oldTaskPids
def getTimeSec():
    ct=time.time()
    return int(ct)
def strToSec(strtime):
    timeArray=time.strptime(strtime,'%Y-%m-%d %H:%M:%S')
    return int(time.mktime(timeArray))
def getStrTime(Sec):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(Sec))
def execcmd(filename):
    return os.system('python %s' % workpath+filename)
def getTaskStatus(task):
    status='init'
    taskStatusPath=workpath+'/taskStatus/'+task+'.status'
    try:
        with open(taskStatusPath,'r') as f:
            status = f.readline()
    except Exception as e:
        logger.error('error:[%s]' % repr(e))
        logger.info('set up %s' % taskStatusPath )
        with open(taskStatusPath,'w') as f:
            f.write('init')
    return status
def initTaskStatus(tasks):
    status='init'
    taskStatusPath=workpath+'/taskStatus/'
    for key in tasks.keys():
        #ret,oldStatus=commands.getstatusoutput("cat %s" % taskStatusPath+key+'.status')
        with open(taskStatusPath + key +'.status','w') as w:
            w.write(status)
def main():
   #将pid写入文件
    with open(workpath+'/cronMain.pid','w') as f:
        f.write(str(os.getpid()))
    #下一个工作日记录文件
    nextworkDatePath=workpath+'/'+config.nextworkDatePath
    #当日工作记录文件
    nowworkDatePath=workpath+'/'+config.workDatePath
    logger.info('init task')
    初始化任务
    init_tasks,todayTasks,oldTaskPids=initTasks()
    initTaskStatus(init_tasks)
    while True:
        nowSec=getTimeSec()
        instantiationTime=getToday()+ ' ' + config.instantiationTime
        logger.info("nowSec=[%d],nowTime=[%s]" %(nowSec,getStrTime(nowSec)))
        if strToSec(instantiationTime) <= nowSec:
            logger.info('nowTime=[%s] > instantiationTime=[%s],start next_task_instantiation()' %(getStrTime(nowSec),instantiationTime))
            next_task_instantiation() #生成下一个工作日任务
        nextworkDate=os.popen('cat %s' % nextworkDatePath).readlines()
        if strToSec(nextworkDate[0]+' '+ '00:00:00') <= nowSec: #刷新任务列表
            with open(nowworkDatePath,'w') as f:
                f.wrtie(nextworkDate)
            with open(nextworkDatePath,'w') as w:
                w.write(get_next_date())
            logger.info('nowwork=[%s],nextwork=[%s],init tasks'%(nextworkDate,get_next_date()))
            init_tasks,todayTasks,oldTaskPids=initTasks() 
            initTaskStatus(init_tasks)
        for key in todayTasks.keys(): #轮询任务
            runtime= todayTasks[key].get('rundate')+' '+todayTasks[key].get('runtime')
            runSec=strToSec(runtime)
            logger.info('task:[%s],runtime=[%s],runSec=[%d]' % (key,runtime,runSec))
            if runSec <= nowSec:
                taskStatus=getTaskStatus(key) #获取任务状态文件中的值
                todayTasks[key]['status']=taskStatus
                logger.info('task=[%s],status=[%s]' %(key,taskStatus))
                if todayTasks[key].get('status') in ['running','finished'] : #任务已完成或者正在进行中
                    logger.info('task:[%s],is running or finished.' % key)
                    continue
                else:
                    oldtaskpid=oldTaskPids.get(key) #任务未进行，先获取pid文件中pid
                    if oldtaskpid:
                        ret=kill(oldtaskpid)# 执行kill
                        if ret == 1:
                            logger.warn('killed error pid=[%s]' % oldtaskpid )
                        else:
                            logger.info('killed success! pid=[%s]' % oldtaskpid)
                    else:
                        logger.warn('oldtaskpid=[%s]' % oldtaskpid) #没有pid 无需执行
                logger.info('task:[%s],filename:[%s],start up' %(key,todayTasks[key].get('filename')))
                try:
                    th=thread.Thread(execcmd,logger,key,todayTasks[key].get('filename')) # 启动线程
                    th.start() #任务执行
                    todayTasks[key]['status']='running' #将任务状态变更为运行中
                except Exception as e:
                    logger.error('error:[%s]' % repr(e))
            else:
                logger.info('task:[%s],time is not yet available.' % key) #任务没有到运行时间
        logger.info('waitTime=[%d]' % waitTime)
        sleep(waitTime) #还行sleep 默认 60秒
if __name__ == '__main__':
    
    mainPath=os.getcwd()+ '/cronMain.pid'
    with open(mainPath,'r') as f:
        mainPid=f.readline()
    is_Pid_exist=os.popen('ps -ef| grep %s | grep -v grep' % mainPid).readlines()
    if mainPid and is_Pid_exist: # cronMain 存在不运行执行；
        logger.warn('cronMain=[%s],process=[%s]' %(mainPid,is_Pid_exist))
        logger.warn("cronMain is running.not allow start-up!") 
        sys.exit(1)
    else:
        logger.info("cronMain is not running.allow start-up.") #没有可以执行
        main()

test.py 思路：
1.test.py 执行开始，想将pid写入文件中；
2.将running 写入状态文件中
3.执行成功后，将finished 写入状态文件；
4.执行错误后，将error，写入状态文件中；

pid 文件命名规则，是根据 task.ini文件中的[]中的字符串取值命名的，例如 TEST1.pid
status文件命名规则同上，例如 TEST1.status