python之实现每日任务调度功能
由于项目需要,有许多开发的小程序,需要每日定时执行,但是又不想用linux自用的crontab功能,所以试着项目中的批量调度编排模式,手写了python版的任务调度。
目录结构
pycrontab
+ bin --存放task配置的任务程序
+ +exec.py -- 任务启动程序
+ log
+ tasks --任务配置信息、实例化配置信息
+ tools --工具类
+ +logger.py --日志封装类
+ +thread.py-- 线程封装类
+ config
+ +config.py --系统配置信息
+ pid -- 记录执行任务程序的进程pid
+ taskStatus -- 记录执行任务程序的执行状态
+ workDate --记录当前工作日文件
+ nextworkDate -- 记录下一个工作日文件
+ cronMain.py --任务调度程序
+ cronMain.pid -- 任务调度启动的进程pid
初始化文件 task.ini
[TEST1]
id=1
runtime=21:00:00
filename=/bin/test/test1.py
calltype=python
max_num=1
isexecute=ON
目前实现的功能:
1.配置化支持每日任务的编排、调度、定时实例化;
2.多线程任务调度;
3.兼容p2p3;
4.按照任务的种类进行日志分割;
5.任务执行前,对任务运行情况进行检查并kill;
6.主程序启动检查(仅允许一个主程序启动);
7.支持多种语言的程序执行调度;
8.配置化的支持任务执行循环频率;
cronMain.py
from config import config
from tools import logger,thread
import os
import datetime
import json
import signal
import time
import sys
from time import sleep
version=sys.version.split(' ')
#p2.p3兼容设置
if version[0][0] == '2':
import ConfigParser
import commands
elif version[0][0] == '3:
import configoarer as ConfigParser
import subprocess as commands
else:
sys.exit(1)
waitTime=config.waittime
init_tasks={}
workpath=os.getcwd()
today=str(datetime.date.today())
logger=logger.Logger(__name__,workpath+'/log/main.log').getLogger()
def getToday():
return str(datetime.date.today())
def get_next_date():
today= datetime.date.today()
return str(today + datetime.timedelta(days=1))
def get_next_next_date():
today= datetime.date.today()
return str(today + datetime.timedelta(days=2))
def readConfig():
tasks={}
readconfig=ConfigParser.ConfigParser()
readconfig.read(workpath+'/tasks/tasks.ini')
sections=readconfig.sections()
for section in sections:
tasks[section]=readconfig.items(section)
return tasks
def writeConfig(tasks,next_day):
logger.info('next_day=[%s]' % next_day)
try:
for key in tasks.keys():
logger.info(tasks[key])
writeconfig=ConfigParser.ConfigParser()
writeconfig.add_section(key)
for v in tasks[key]:
writeconfig.set(key,v[0],v[1])
writeconfig.set(key,'rundate',next_day)
writeconfig.set(key,'enddate',get_next_next_date())
writeconfig.set(key,'status','init')
filename=workpath+'/tasks/'+next_day+'_'+key+'_task.ini'
logger.info("instantiation-->"+filename)
with open(filename,'w') as f:
writeconfig.write(f)
except Exception as e:
logger.error('error:[%s]' % repr(e))
def next_task_instantiation():
try:
next_day=get_next_date()
tasks=init_tasks
if not tasks:
raise OSError
logger.info('tasks=[%s]' % json.dumps(tasks))
writeConfig(tasks,next_day)
with open(workpath+'/nextworkDate','w') as wf:
wf.write(next_day)
except OSError:
logger.error("Can't open config file: tasks.ini")
def getTodayTasks():
#read init tasks.ini
tasks=init_tasks
today=getToday()
todayTasks={}
for key in tasks.keys():
TaskFile=workpath+'/tasks/'+today+'_'+key+'_task.ini'
if not os.path.exists(TaskFile):
logger.error("Taskfile=[%s] is not exists."% TaskFile )
tmp=list(tasks[key])
tmp.append(('init','tasks.ini'))
tmp.append(('rundate',today))
tmp.append(('enddate',get_next_date()))
tmp.append(('status','init'))
todayTasks[key]=listTodict(tmp)
else:
logger.info("Taskfile=[%s] is exists."% TaskFile)
readconfig=ConfigParser.ConfigParser()
readconfig.read(TaskFile)
sections=readconfig.sections()
for section in sections:
tasks[section]=readconfig.items(section)
tmp=list(tasks[key])
todayTasks[key]=listTodict(tmp)
return todayTasks
def listTodict(listdate):
task_dict={}
for i in listdate:
task_dict[i[0]]=i[1]
return task_dict
def kill(pid):
import signal
pname=os.popen('ps -ef| grep %s | grep -v grep' % pid)
logger.info("kill pid:[%s]" % pname.readlines())
try:
ret=os.kill(int(pid),signal.SIGKILL)
logger.info("killed pid=[%s],ret=[%s]" % (pid ,str(ret)))
return ret
except Exception as e:
logger.error('error:[%s]' % repr(e))
return 1
def getOldTaskPids():
TaskPids={}
fileslist=os.listdir(workpath+'/pid')
pid=''
for f in fileslist:
tmpPath=workpath+'/pid/'+f
pid= os.popen('cat %s' % tmpPath)
appCode=f.split('.')[0]
TaskPids[appCode]=pid.readlines()[0]
logger.info('TaskPids=[%s]' % json.dumps(TaskPids))
return TaskPids
def initTasks():
global init_tasks
init_tasks=readConfig()
today=getToday()
with open(workpath+'/'+config.workDatePath,'w') as f:
f.write(today)
with open(workpath+'/'+config.nextworkDatePath,'w') as f:
f.write(get_next_date())
logger.info("initTasks=[%s]" % json.dumps(init_tasks))
todayTasks=getTodayTasks()
logger.info("todayTask=[%s]" % json.dumps(todayTasks))
oldTaskPids=getOldTaskPids()
logger.info('oldTaskPids=[%s]' % json.dumps(oldTaskPids))
return init_tasks,todayTasks,oldTaskPids
def getTimeSec():
ct=time.time()
return int(ct)
def strToSec(strtime):
timeArray=time.strptime(strtime,'%Y-%m-%d %H:%M:%S')
return int(time.mktime(timeArray))
def getStrTime(Sec):
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(Sec))
def execcmd(filename):
return os.system('python %s' % workpath+filename)
def getTaskStatus(task):
status='init'
taskStatusPath=workpath+'/taskStatus/'+task+'.status'
try:
with open(taskStatusPath,'r') as f:
status = f.readline()
except Exception as e:
logger.error('error:[%s]' % repr(e))
logger.info('set up %s' % taskStatusPath )
with open(taskStatusPath,'w') as f:
f.write('init')
return status
def initTaskStatus(tasks):
status='init'
taskStatusPath=workpath+'/taskStatus/'
for key in tasks.keys():
#ret,oldStatus=commands.getstatusoutput("cat %s" % taskStatusPath+key+'.status')
with open(taskStatusPath + key +'.status','w') as w:
w.write(status)
def main():
#将pid写入文件
with open(workpath+'/cronMain.pid','w') as f:
f.write(str(os.getpid()))
#下一个工作日记录文件
nextworkDatePath=workpath+'/'+config.nextworkDatePath
#当日工作记录文件
nowworkDatePath=workpath+'/'+config.workDatePath
logger.info('init task')
初始化任务
init_tasks,todayTasks,oldTaskPids=initTasks()
initTaskStatus(init_tasks)
while True:
nowSec=getTimeSec()
instantiationTime=getToday()+ ' ' + config.instantiationTime
logger.info("nowSec=[%d],nowTime=[%s]" %(nowSec,getStrTime(nowSec)))
if strToSec(instantiationTime) <= nowSec:
logger.info('nowTime=[%s] > instantiationTime=[%s],start next_task_instantiation()' %(getStrTime(nowSec),instantiationTime))
next_task_instantiation() #生成下一个工作日任务
nextworkDate=os.popen('cat %s' % nextworkDatePath).readlines()
if strToSec(nextworkDate[0]+' '+ '00:00:00') <= nowSec: #刷新任务列表
with open(nowworkDatePath,'w') as f:
f.wrtie(nextworkDate)
with open(nextworkDatePath,'w') as w:
w.write(get_next_date())
logger.info('nowwork=[%s],nextwork=[%s],init tasks'%(nextworkDate,get_next_date()))
init_tasks,todayTasks,oldTaskPids=initTasks()
initTaskStatus(init_tasks)
for key in todayTasks.keys(): #轮询任务
runtime= todayTasks[key].get('rundate')+' '+todayTasks[key].get('runtime')
runSec=strToSec(runtime)
logger.info('task:[%s],runtime=[%s],runSec=[%d]' % (key,runtime,runSec))
if runSec <= nowSec:
taskStatus=getTaskStatus(key) #获取任务状态文件中的值
todayTasks[key]['status']=taskStatus
logger.info('task=[%s],status=[%s]' %(key,taskStatus))
if todayTasks[key].get('status') in ['running','finished'] : #任务已完成或者正在进行中
logger.info('task:[%s],is running or finished.' % key)
continue
else:
oldtaskpid=oldTaskPids.get(key) #任务未进行,先获取pid文件中pid
if oldtaskpid:
ret=kill(oldtaskpid)# 执行kill
if ret == 1:
logger.warn('killed error pid=[%s]' % oldtaskpid )
else:
logger.info('killed success! pid=[%s]' % oldtaskpid)
else:
logger.warn('oldtaskpid=[%s]' % oldtaskpid) #没有pid 无需执行
logger.info('task:[%s],filename:[%s],start up' %(key,todayTasks[key].get('filename')))
try:
th=thread.Thread(execcmd,logger,key,todayTasks[key].get('filename')) # 启动线程
th.start() #任务执行
todayTasks[key]['status']='running' #将任务状态变更为运行中
except Exception as e:
logger.error('error:[%s]' % repr(e))
else:
logger.info('task:[%s],time is not yet available.' % key) #任务没有到运行时间
logger.info('waitTime=[%d]' % waitTime)
sleep(waitTime) #还行sleep 默认 60秒
if __name__ == '__main__':
mainPath=os.getcwd()+ '/cronMain.pid'
with open(mainPath,'r') as f:
mainPid=f.readline()
is_Pid_exist=os.popen('ps -ef| grep %s | grep -v grep' % mainPid).readlines()
if mainPid and is_Pid_exist: # cronMain 存在不运行执行;
logger.warn('cronMain=[%s],process=[%s]' %(mainPid,is_Pid_exist))
logger.warn("cronMain is running.not allow start-up!")
sys.exit(1)
else:
logger.info("cronMain is not running.allow start-up.") #没有可以执行
main()
test.py 思路:
1.test.py 执行开始,想将pid写入文件中;
2.将running 写入状态文件中
3.执行成功后,将finished 写入状态文件;
4.执行错误后,将error,写入状态文件中;
pid 文件命名规则,是根据 task.ini文件中的[]中的字符串取值命名的,例如 TEST1.pid
status文件命名规则同上,例如 TEST1.status