从配置文件读取配置参数,并每天定时执行日常任务、每周一定时执行周任务,匹配符合要求的pos数据。
定时任务运行日志:
import os
import datetime
import re
import shutil
import time
import threading
import schedule
from dateutil import parser
# 全局变量:配置文件
glob_config = "configss.txt"
def txt_to_dict(filename):
with open(filename, 'r', encoding='utf-8') as f:
data_dict = {}
for line in f:
# 忽略空行和注释行
if line.strip() == '' or line.startswith('#'):
continue
key, value = line.strip().split('=')
data_dict[key] = value
return data_dict
def copy_exchange(src_path, dst_path, pattern_start, pattern_end):
# 示例:将2023年10月19日12时(UTC时间)转换为GPS周和周内秒
utc_time_s = datetime.datetime.strptime(pattern_start, '%Y-%m-%d %H:%M:%S')
utc_time_e = datetime.datetime.strptime(pattern_end, '%Y-%m-%d %H:%M:%S')
# GPS周起始时间(1980年1月6日)
gps_epoch = datetime.datetime(1980, 1, 6)
# 计算时间差
time_difference_s = utc_time_s - gps_epoch
time_difference_e = utc_time_e - gps_epoch
# 计算总秒数和周数,gps时间比utc时间快了18秒
total_seconds_s = time_difference_s.total_seconds() + 18
gps_week_s = total_seconds_s // (7 * 24 * 3600)
gps_week_seconds_s = total_seconds_s % (7 * 24 * 3600)
pattern_start_ss = str(int(gps_week_s)) + " " + str(gps_week_seconds_s)
total_seconds_e = time_difference_e.total_seconds() + 18
gps_week_e = total_seconds_e // (7 * 24 * 3600)
gps_week_seconds_e = total_seconds_e % (7 * 24 * 3600)
pattern_end_ee = str(int(gps_week_e)) + " " + str(gps_week_seconds_e)
# 文件命名
now = datetime.datetime.now()
p = pattern_start.split()
fn = re.sub(r'-', '', p[0])
fnn = re.sub(r':', '', p[1])
# 区分不同环境的pos文件
if "PROD_V2_N" in src_path:
print("PROD_V2_N环境的SSR文件")
file_name = "PROD_N_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
elif "PROD_V2" in src_path:
print("PROD_V2环境的SSR文件")
file_name = "PROD_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
elif "TEST_V2_P" in src_path:
print("TEST_V2_P环境的SSR文件")
file_name = "TEST_P_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
elif "TEST_V2" in src_path:
print("TEST_V2环境的SSR文件")
file_name = "TEST_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
else:
print("非PROD和TEST环境的SSR文件")
file_name = fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
# 复制文件
dst_path_c = os.path.join(dst_path, file_name)
shutil.copy2(src_path, dst_path_c)
dst_w_path = os.path.join(dst_path, 'N_' + file_name)
# 写数据
with open(dst_path_c, 'r') as dst, open(dst_w_path, 'w') as dst_w:
# 先写入文件前两行
lineTwo = dst.readlines()[:2]
dst_w.writelines(lineTwo)
with open(dst_path_c, 'r') as dst, open(dst_w_path, 'a+') as dst_w:
# 再匹配
# 默认不写入文件
writing = False
num_count = 0
for line in dst:
# 正则匹配,起始行
if re.search(pattern_start_ss, line):
writing = True
if writing:
num_count += 1
dst_w.write(line)
# 正则匹配,截止行
if re.search(pattern_end_ee, line):
writing = False
break
if num_count == 0:
print(r'当前时间:%s,没有数据匹配' % (now.strftime('%Y-%m-%d %H:%M:%S')))
else:
print(r'当前时间:%s,总共%s行写入完成,起始于:%s,截止于:%s' % (
now.strftime('%Y-%m-%d %H:%M:%S'), num_count, pattern_start, pattern_end))
if dst_path_c:
os.remove(dst_path_c)
def check_date_format(date_string):
# try:
# parser.parse(date_string)
# return True
# except ValueError:
# return False
# 判断是否符合年月日格式,格式为YYYY-MM-DD
date_pattern = r'^\d{4}-\d{2}-\d{2}$'
match_date = date_string.split()
# print(match_date[0])
if re.match(date_pattern, match_date[0]):
return True
else:
return False
# 日常任务
def job_test():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_test = data.get('origin_path_test', 'null')
dst_path = data.get('dst_path', 'null')
date_start = data.get('date_start', 'null')
date_end = data.get('date_end', 'null')
# 根据路径决定是否执行任务
if origin_path_test:
# 判断时间日期格式
if check_date_format(date_start):
print(f'{date_start}:年月日时分秒格式')
else:
print(f'{date_start}:时分秒格式')
# 获取当前日期
today = datetime.date.today()
# 获取前一天的日期
yesterday = today - datetime.timedelta(days=1)
# 拼接起始和截止时间
date_start = str(yesterday) + " " + date_start
date_end = str(yesterday) + " " + date_end
copy_exchange(origin_path_test, dst_path, date_start, date_end)
else:
print(f'当前环境无任务执行')
# 日常任务-同生产频点
def job_test_p():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_test_p = data.get('origin_path_test_p', 'null')
dst_path = data.get('dst_path', 'null')
date_start = data.get('date_start', 'null')
date_end = data.get('date_end', 'null')
# 根据路径决定是否执行任务
if origin_path_test_p:
# 判断时间日期格式
if check_date_format(date_start):
print(f'{date_start}:年月日时分秒格式')
else:
print(f'{date_start}:时分秒格式')
# 获取当前日期
today = datetime.date.today()
# 获取前一天的日期
yesterday = today - datetime.timedelta(days=1)
# 拼接起始和截止时间
date_start = str(yesterday) + " " + date_start
date_end = str(yesterday) + " " + date_end
copy_exchange(origin_path_test_p, dst_path, date_start, date_end)
else:
print(f'当前环境无任务执行')
# 日常任务-新域名
def job_prod_n():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_prod_n = data.get('origin_path_prod_n', 'null')
dst_path = data.get('dst_path', 'null')
date_start = data.get('date_start', 'null')
date_end = data.get('date_end', 'null')
# 根据路径决定是否执行任务
if origin_path_prod_n:
# 判断时间日期格式
if check_date_format(date_start):
print(f'{date_start}:年月日时分秒格式')
else:
print(f'{date_start}:时分秒格式')
# 获取当前日期
today = datetime.date.today()
# 获取前一天的日期
yesterday = today - datetime.timedelta(days=1)
# 拼接起始和截止时间
date_start = str(yesterday) + " " + date_start
date_end = str(yesterday) + " " + date_end
copy_exchange(origin_path_prod_n, dst_path, date_start, date_end)
else:
print(f'当前环境无任务执行')
# 日常任务
def job_prod():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_prod = data.get('origin_path_prod', 'null')
dst_path = data.get('dst_path', 'null')
date_start = data.get('date_start', 'null')
date_end = data.get('date_end', 'null')
# 根据路径决定是否执行任务
if origin_path_prod:
# 判断时间日期格式
if check_date_format(date_start):
print(f'{date_start}:年月日时分秒格式')
else:
print(f'{date_start}:时分秒格式')
# 获取当前日期
today = datetime.date.today()
# 获取前一天的日期
yesterday = today - datetime.timedelta(days=1)
# 拼接起始和截止时间
date_start = str(yesterday) + " " + date_start
date_end = str(yesterday) + " " + date_end
copy_exchange(origin_path_prod, dst_path, date_start, date_end)
else:
print(f'当前环境无任务执行')
# 周任务
def job_test_weekly():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_test = data.get('origin_path_test', 'null')
dst_path = data.get('dst_path', 'null')
date_start_weekly = data.get('date_start_weekly', 'null')
date_end_weekly = data.get('date_end_weekly', 'null')
# 根据路径决定是否执行任务
if origin_path_test:
# 获取当前日期
today = datetime.date.today()
idx = (today.weekday() + 1) % 7 # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
# 获取上周五、六、日的日期
fri = today - datetime.timedelta(idx + 2)
sun = today - datetime.timedelta(idx + 1)
sat = today - datetime.timedelta(idx)
# 拼接周六的起始和截止时间
date_start_o = str(fri) + " " + date_start_weekly
date_end_o = str(sun) + " " + date_end_weekly
# 拼接周日的起始和截止时间
date_start_t = str(sun) + " " + date_start_weekly
date_end_t = str(sat) + " " + date_end_weekly
copy_exchange(origin_path_test, dst_path, date_start_o, date_end_o)
copy_exchange(origin_path_test, dst_path, date_start_t, date_end_t)
else:
print(f'当前环境无任务执行')
# 周任务-同生产频点
def job_test_p_weekly():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_test_p = data.get('origin_path_test_p', 'null')
dst_path = data.get('dst_path', 'null')
date_start_weekly = data.get('date_start_weekly', 'null')
date_end_weekly = data.get('date_end_weekly', 'null')
if origin_path_test_p:
# 获取当前日期
today = datetime.date.today()
idx = (today.weekday() + 1) % 7 # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
# 获取上周五、六、日的日期
fri = today - datetime.timedelta(idx + 2)
sun = today - datetime.timedelta(idx + 1)
sat = today - datetime.timedelta(idx)
# 拼接周六的起始和截止时间
date_start_o = str(fri) + " " + date_start_weekly
date_end_o = str(sun) + " " + date_end_weekly
# 拼接周日的起始和截止时间
date_start_t = str(sun) + " " + date_start_weekly
date_end_t = str(sat) + " " + date_end_weekly
copy_exchange(origin_path_test_p, dst_path, date_start_o, date_end_o)
copy_exchange(origin_path_test_p, dst_path, date_start_t, date_end_t)
else:
print(f'当前环境无任务执行')
# 周任务
def job_prod_weekly():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_prod = data.get('origin_path_prod', 'null')
dst_path = data.get('dst_path', 'null')
date_start_weekly = data.get('date_start_weekly', 'null')
date_end_weekly = data.get('date_end_weekly', 'null')
if origin_path_prod:
# 获取当前日期
today = datetime.date.today()
idx = (today.weekday() + 1) % 7 # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
# 获取上周五、六、日的日期
fri = today - datetime.timedelta(idx + 2)
sun = today - datetime.timedelta(idx + 1)
sat = today - datetime.timedelta(idx)
# 拼接周六的起始和截止时间
date_start_o = str(fri) + " " + date_start_weekly
date_end_o = str(sun) + " " + date_end_weekly
# 拼接周日的起始和截止时间
date_start_t = str(sun) + " " + date_start_weekly
date_end_t = str(sat) + " " + date_end_weekly
copy_exchange(origin_path_prod, dst_path, date_start_o, date_end_o)
copy_exchange(origin_path_prod, dst_path, date_start_t, date_end_t)
else:
print(f'当前环境无任务执行')
# 周任务-新域名
def job_prod_n_weekly():
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取特定键值对的值
origin_path_prod_n = data.get('origin_path_prod_n', 'null')
dst_path = data.get('dst_path', 'null')
date_start_weekly = data.get('date_start_weekly', 'null')
date_end_weekly = data.get('date_end_weekly', 'null')
if origin_path_prod_n:
# 获取当前日期
today = datetime.date.today()
idx = (today.weekday() + 1) % 7 # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
# 获取上周五、六、日的日期
fri = today - datetime.timedelta(idx + 2)
sun = today - datetime.timedelta(idx + 1)
sat = today - datetime.timedelta(idx)
# 拼接周六的起始和截止时间
date_start_o = str(fri) + " " + date_start_weekly
date_end_o = str(sun) + " " + date_end_weekly
# 拼接周日的起始和截止时间
date_start_t = str(sun) + " " + date_start_weekly
date_end_t = str(sat) + " " + date_end_weekly
copy_exchange(origin_path_prod_n, dst_path, date_start_o, date_end_o)
copy_exchange(origin_path_prod_n, dst_path, date_start_t, date_end_t)
else:
print(f'当前环境无任务执行')
def run_threading(job_func):
# 多线程并行运行
job_thread = threading.Thread(target=job_func)
job_thread.start()
if __name__ == '__main__':
# 读取配置文件,加载字典
data = txt_to_dict(glob_config)
# 获取运行时间
# run_time = "16:04"
run_time = data.get('run_time', 'null')
run_time_weekly = data.get('run_time_weekly', 'null')
# 创建调度器
schedule_daily = schedule.Scheduler()
# 每天定时调度日常任务
schedule_daily.every().day.at(run_time).do(run_threading, job_test)
schedule_daily.every().day.at(run_time).do(run_threading, job_test_p)
schedule_daily.every().day.at(run_time).do(run_threading, job_prod)
schedule_daily.every().day.at(run_time).do(run_threading, job_prod_n)
# schedule.every(2).minutes.do(run_threading, job_test) # 每两分钟
# schedule.every(2).minutes.do(run_threading, job_prod) # 每两分钟
# 创建调度器
schedule_weekly = schedule.Scheduler()
# 每周的星期一定时调度周任务
schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_test_weekly)
schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_test_p_weekly)
schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_prod_weekly)
schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_prod_n_weekly)
# 立即执行所有任务
schedule_daily.run_all()
schedule_weekly.run_all()
while True:
schedule_daily.run_pending()
schedule_weekly.run_pending()
time.sleep(1)
config配置文件如下:
# 日常任务执行时间-北京时间
run_time=08:05
# 读取的原文件路径,默认test、test_p、prod、prod_n四个环境,若停止监控该环境,只需将该值置空
#origin_path_test=E:/python/pythonProject1/threading_test/N_TEST_20231105.pos
origin_path_test=E:/data/0925/move.pos
origin_path_test_p=E:/data/0925/move.pos
# 存放文件的路径
dst_path=E:/python/pythonProject1/threading_test/
# 日常的起始匹配字符串:UTC时间:2023-11-06 08:27:08和08:27:08格式均支持,date_start和date_end必须格式相同
#date_start=2023-11-06 08:27:08
date_start=12:00:00
# 日常的截止匹配字符串:UTC时间
#date_end=2023-11-06 09:27:08
date_end=23:59:59
# 周任务执行时间-北京时间
run_time_weekly=00:05
# 周末的起始匹配字符串:UTC时间
date_start_weekly=16:00:00
# 周末的截止匹配字符串:UTC时间
date_end_weekly=15:59:59