参考:https://my.oschina.net/lionets/blog/796438
问题发生:
我根据python的TimedRotatingFileHandler写了一个每天自动切割日志模块,但是第二天报logger error,提示我另一个程序正在使用此文件,进程无法访问。
问题原因:
python的TimedRotatingFileHandler在多进程下因为每个进程都会调用一次 doRollover,就可能会发生像一个进程已经 rollover 完成,但是下一个进程把之前的 error.log 又给删掉了之类的问题。
问题解决:
代码可直接复用,但注意不支持utc时间,且只适用于按天分割
# 解决logging模块多进程安全问题
class MultiProcessSafeDailyRotatingFileHandler(BaseRotatingHandler):
"""
- Multi process safe
- Rotate at midnight only
- Utc not supported
"""
def __init__(self, filename, encoding=None, delay=False, utc=False, backupCount=30, **kwargs):
"""
Args:
filename: 日志路径
backupCount: 日志保存天数(按天分割)
"""
self.utc = utc
self.suffix = "%Y-%m-%d.log"
self.baseFilename = filename
self.backupCount = backupCount
self.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", re.ASCII)
self.currentFileName = self._compute_fn()
BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
def shouldRollover(self, record):
if self.currentFileName != self._compute_fn():
return True
return False
def doRollover(self):
if self.stream:
self.stream.close()
self.stream = None
self.currentFileName = self._compute_fn()
def _compute_fn(self):
return self.baseFilename + "." + time.strftime(self.suffix, time.localtime())
def _open(self):
if self.encoding is None:
stream = open(self.currentFileName, self.mode)
else:
stream = codecs.open(self.currentFileName, self.mode, self.encoding)
if os.path.exists(self.baseFilename):
try:
os.remove(self.baseFilename)
except OSError:
pass
try:
os.symlink(self.currentFileName, self.baseFilename)
# 调用删除函数
if self.backupCount > 0:
for s in self.getFilesToDelete():
os.remove(s)
except OSError:
pass
return stream
# 过期日志删除函数
def getFilesToDelete(self):
"""
Determine the files to delete when rolling over.
"""
dirName, baseName = os.path.split(self.baseFilename)
fileNames = os.listdir(dirName)
result = []
prefix = baseName + "."
plen = len(prefix)
for fileName in fileNames:
if fileName[:plen] == prefix:
suffix = fileName[plen:]
if self.extMatch.match(suffix):
result.append(os.path.join(dirName, fileName))
if len(result) < self.backupCount:
result = []
else:
result.sort(reverse=True)
result = result[self.backupCount:]
return result
日志调用函数:
# 日志生成函数
def get_log(file_name):
"""
Args:
file_name: 日志路径
Returns:
logger: logger对象
"""
logger = logging.getLogger(file_name)
logger.setLevel(logging.INFO)
log_path = str(pathlib.Path(__file__).parent.parent.resolve()) + os.path.sep + 'log' + os.path.sep + file_name
exist = os.path.exists(log_path)
if not exist:
os.makedirs(log_path)
log_file = log_path + os.path.sep + file_name
file_handler = MultiProcessSafeDailyRotatingFileHandler(filename=log_file, encoding='utf8', backupCount=30)
# * 定义日志输出格式
file_handler.setFormatter(
logging.Formatter(
"[%(asctime)s] [%(process)d] [%(levelname)s] - %(module)s.%(funcName)s - [%(filename)s:%(lineno)d] - %(message)s"))
logger.addHandler(file_handler)
return logger