以python代码为例,与他人交流后的一些想法,先重构一个自己以前写的代码.
重构前代码, 业务逻辑写在了一起,被评价为 脚本
def sync_record_hourly():
# 获取post过来的文件名称,参数中获取,指定name为file
upload_files = request.files.getlist('file')
if not upload_files:
return {"code": 500, "err": "Please upload the file with 'file' as the key"}
for file in upload_files:
if file and allowed_file(file.filename):
load = Access.load(file)
count = 0
title_flag = True
title = []
time_c = ''
while True:
content_line = file.readline()
if len(content_line) == 0:
break
con = content_line.decode(encoding="utf-8").replace(os_linesep, '') # type: str
if not con:
continue
suffix_name = file.filename.rsplit('.', 1)[1]
if suffix_name == 'txt':
con_list = [item.strip() for item in con.split('\t') if item.strip()]
else:
con_list = [item.strip() for item in con.split(',') if item.strip()]
if title_flag:
if 'data_traffic_real' not in con_list or 'user_id' not in con_list:
break
# 保存文件头部信息
title = con_list
title_flag = False
continue
time_c = datetime.strptime(con_list[title.index('update_time')], '%Y-%m-%d %H:%M:%S').replace(minute=0, second=0, microsecond=0)
if suffix_name == 'txt':
ips = con_list[title.index('ips')].split(',') if 'ips' in title else []
else:
ips = con_list[: title.index('ips') - 1: -1] if 'ips' in title else []
account = con_list[title.index('account')] if 'account' in title else None
user = con_list[title.index('user_id')]
real_traffic = float(con_list[title.index('data_traffic_real')]) * 1024
doc = {
'account': account,
'time': time_c,
'ips': ips,
'user': user,
'traffic': real_traffic,
}
hourly_fw_s = RecordWfei.get_traffic(start=time_c.replace(hour=0), end=time_c, user=user, identified=False)
if hourly_fw_s:
inc_traffic = real_traffic - sum([item.traffic for item in hourly_fw_s])
doc['traffic'] = inc_traffic
hourly_wfei = RecordWfei(doc)
hourly_wfei.save()
record_influx = RecordInfluxdb.get_traffic(time=time_c, user=user)
if record_influx:
doc['traffic'] += float(record_influx.traffic)
doc['ips'].extend(list(record_influx.ips))
hourly = RecordHourly(doc)
hourly.save()
count += 1
logger.info(f'Saved {count} wfei access hourly records at {time_c}.')
return {"code": 200, "message": "Data synchronization successful"}
重构, 封装出Access对象(对象要有属性和行为,对象操作自己,完成功能)
#!/usr/bin/python3
# @Time : 2021/3/18下午 05:11
# @Author : Ryan
# @Description :
# -*- coding: utf-8 -*-
def clear_tail(content):
if '\r\n' in content:
return content.replace('\r\n', '')
if '\r' in content:
return content.replace('\r', '')
if '\n' in content:
return content.replace('\n', '')
class Txt:
""" csv文件使用 tab 分隔 """
split_str = '\t'
# ips 使用 ; 分隔
ips_split_str = ';'
class Csv:
""" csv文件使用 , 分隔 """
split_str = ','
# ips 使用 ; 分隔
ips_split_str = ';'
class Access:
"""
eager loading
一次性把文件的内容全部读取, 后通过迭代器得到每一行数据;
"""
n = 0
rows = []
fields = []
def __init__(self, file):
rows = []
header = None
# 使用with, 避免忽略了关闭资源
with file as f:
while True:
row = f.readline().decode(encoding="utf-8")
if len(row) == 0:
break
row = clear_tail(row)
if not row.strip():
continue
if not header:
header = row
continue
if len(header.split(self.split_str)) != len(row.split(self.split_str)):
continue
rows.append(row)
self.fields = header.split(self.split_str)
self.rows = rows
@staticmethod
def load(file):
if file.rsplit('.', 1)[1] == 'txt':
return EagerTxt(file)
elif file.rsplit('.', 1)[1] == 'csv':
return EagerCsv(file)
else:
raise NotImplementedError('错误的文件类型')
def __iter__(self):
return self
def __next__(self):
"""定义迭代方法,每次迭代返回字典(头部和数据),类似于数据库"""
# fields = [ 'id', 'name', 'update_time' ]
if self.n >= len(self.rows):
raise StopIteration
cells = self.rows[self.n]
r = {}
for i, field in enumerate(self.fields):
row_con = cells.split(self.split_str)
if len(row_con) != len(self.fields):
continue
if 'ips' == field:
r[field] = row_con[i].replace('[', '').replace(']', '').split(self.ips_split_str)
else:
r[field] = row_con[i]
self.n += 1
return r
class EagerTxt(Access, Txt):
"""初始化将文件的所有内容读到rows和header中"""
pass
class EagerCsv(Access, Csv):
"""初始化将文件的所有内容读到rows和header中"""
pass
class LazyAccess:
"""
lazy loading
懒加载,需要数据的时候再从文件中读取数据
"""
def __init__(self, file):
self.heads = []
self.file = open(file, 'r')
self.read_head()
def __iter__(self):
return self
def read_head(self):
# 先读取头部,指针指向下一行,迭代的时候就直接读到数据区了,避免迭代第一次没有数据
while True:
readline = self.file.readline().decode(encoding="utf-8")
readline = clear_tail(readline)
if readline:
self.heads = readline.split(self.split_str)
break
def __next__(self):
r = {}
# d=读取每一行数据
readline = self.file.readline().decode(encoding="utf-8") # type: str
# 是否读完了
if len(readline) == 0:
raise StopIteration
# 去掉换行符
readline = clear_tail(readline)
# 如果去掉换行符就没有内容了就说明没有数据了
if not readline:
raise StopIteration
# 按规定切割
row = readline.split(self.split_str)
# 如果没有头部,则赋值这个头部
if not self.heads:
self.heads = row
else:
# 如果存在头部,则说明读到数据区了 开始拼接数据 r
for i, item in enumerate(self.heads):
if item == 'ips':
# 因为ips是按 ; 分隔的,所以去掉 [ ],得到真正得list
r[item] = row[i].replace('[', '').replace(']', '').split(self.ips_split_str)
else:
r[item] = row[i]
return r
def close(self):
"""因为是懒加载模式,避免子类未重写这个方法,而导致资源未释放"""
raise NotImplementedError
@staticmethod
def load(file):
if file.rsplit('.', 1)[1] == 'txt':
return LazyTxt(file)
elif file.rsplit('.', 1)[1] == 'csv':
return LazyCsv(file)
else:
raise NotImplementedError('未知的文件类型')
class LazyTxt(LazyAccess, Txt):
def __init__(self, file):
super().__init__(file)
def close(self):
if self.file:
self.file.close()
class LazyCsv(LazyAccess, Csv):
def __init__(self, file):
super().__init__(file)
def close(self):
if self.file:
self.file.close()
if __name__ == '__main__':
with open('C:\\Users\\ct\\Desktop\\giji\\wfei话单的') as f:
load = Access.load(f)