python(5) : 日志打印及自动压缩清理日志

 my_log.py

# -*- coding: utf-8 -*-

# ====> 日志打印 <==== #

import datetime
import gzip
import logging
import os
import shutil
import time
import traceback

# 创建一个logger
logger = logging.getLogger('MyLogger')

# 设置日志记录级别为DEBUG
logger.setLevel(logging.DEBUG)

# 定义日志文件
log_filename = 'app.log'

# 创建一个handler，用于写入日志文件
fh = logging.FileHandler(log_filename)

# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()

# 定义handler的输出格式
formatter = logging.Formatter(
    '%(asctime)s | [%(process)d] | %(levelname)s | %(filename)s | %(lineno)d | %(funcName)s | %(message)s')

fh.setFormatter(formatter)
ch.setFormatter(formatter)

# 给logger添加handler
logger.addHandler(fh)
logger.addHandler(ch)

# 日志管理

# 历史文件存放目录
history_log_folder = 'history_logs'
# 获取当前日期
current_date = datetime.date.today()
# 设置日志文件保存的天数
max_log_age = 7
# 设置需要压缩的历史日志文件保存的天数
max_archive_age = 30
# 计算7天前和30天前的日期
max_archive_age_date = current_date - datetime.timedelta(days=max_archive_age)
max_log_age_date = current_date - datetime.timedelta(days=max_log_age)

os.makedirs(history_log_folder, exist_ok=True)


# 获取主日志文件中n天之前最后一行
def get_current_log_first_line(file_path):
    with open(file_path, 'r') as file:
        lines = file.readlines()
        for line_number, line in enumerate(reversed(lines), start=1):
            asc_line = len(lines) - line_number + 1
            log_date_str = line[:10]
            try:
                log_date = datetime.datetime.strptime(log_date_str, '%Y-%m-%d').date()
            except Exception:
                continue
            if (current_date - log_date).days >= max_log_age:
                return asc_line


# 清理历史日志压缩文件
def clear_history_file():
    for filename in os.listdir(history_log_folder):
        print(filename[:10])
        history_file_date = datetime.datetime.strptime(filename[:10], "%Y-%m-%d").date()
        if (current_date - history_file_date).days >= max_log_age:
            os.remove(f'{history_log_folder}{os.sep}{filename}')


# 移动日志到历史文件并压缩
def manager_log():
    try:
        logger.info('执行定时任务处理日志...')
        current_log_first_line = get_current_log_first_line(log_filename)
        if current_log_first_line is None:
            logger.info('无待处理的日志, 定时任务处理日志执行完成')
            return None
        logger.info(f'待处理的日志行数:{current_log_first_line}')
        # 复制到历史文件
        logger.info('执行复制到历史文件')
        with open(log_filename, 'r') as file:
            lines = file.readlines()[:current_log_first_line]
        old_log_file = f'{history_log_folder}{os.sep}{max_archive_age_date}.log'
        with open(old_log_file, 'a') as file:
            for line in lines:
                file.write(line)
        # 删除旧日志
        logger.info('执行删除旧日志')
        with open(log_filename, 'r') as fin:
            data = fin.read().splitlines(True)
        with open(log_filename, 'w') as fout:
            fout.writelines(data[current_log_first_line:])
        # 压缩历史文件
        logger.info('执行压缩历史文件')
        with open(old_log_file, 'rb') as f_in, gzip.open(old_log_file + '.gz', 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)
        # 删除历史文件
        logger.info('执行删除历史文件')
        os.remove(old_log_file)
        # 清理
        logger.info('执行清理历史日志压缩文件')
        clear_history_file()
        logger.info(f'定时任务处理日志执行完成, 处理日志行数:{current_log_first_line}')
    except Exception:
        traceback.print_exc()

打印日志

 
from my_log import logger as lg
 
lg.info('hello world')

日志打印样例

2024-08-14 10:39:41,633 | INFO | [1219] | test.py | 1 | main | hello wrold

flask添加定时任务管理日志

# -*- coding: utf-8 -*-

# ====> 主程序 <==== #

from apscheduler.schedulers.background import BackgroundScheduler
from flask import app
from my_log import logger as lg, manager_log


if __name__ == '__main__':
    lg.info('启动管理日志定时任务')
    scheduler = BackgroundScheduler()
    # 每天2点执行
    scheduler.add_job(func=manager_log, trigger='cron', day_of_week='*', hour=2)
    scheduler.start()
    lg.info('管理日志定时任务启动成功')
    # 启动web服务
    app.run(host='0.0.0.0', port=8080, debug=False)

安装依赖

pip install apscheduler  -i https://mirrors.aliyun.com/pypi/simple/ requests
pip install flask -i https://mirrors.aliyun.com/pypi/simple/ requests