背景
系统有个周期读取对应ftp指定目录下文件,进行文件传输的功能,为了功能稳定性测试。需要周期性的生成一批数据在对应目录下。为了满足创建性能需求()可能是多个指定目录创建,创建的文件个数较多,且调度周期间隔短)所以需要快速的增量创建。
使用多线程开发了如下脚本
# coding=utf-8
# @Time : 2022/6/2 16:16
# @Author : qyw
# @File : File_batch.py
# @Desc : 指定目录下 制定时间间隔内 批量创建文件夹和文件
import threading, sys, os
import shutil, time
from datetime import datetime
import math
#创建文件方法,周期性创建爱你 间隔30s
def create_file(file_path_list, file_num):
"""
:param file_path_list: 分配的待创建文件的目录list
:param file_num: 待创建文件数量
:return:
"""
try:
num = 0
while True:
st_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
time_flag = str(time.time())[5:10]
for e_path in file_path_list:
temp_dir_name = os.path.split(e_path)[-1]
for i in range(int(file_num)):
path_new_file = os.path.join(e_path,
temp_dir_name + "_" + str(time_flag) + "_" + str(num) + "_" + str(i))
shutil.copyfile('./demo.txt', path_new_file)
end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
start_time1 = time.strptime(st_time, "%Y-%m-%d %H:%M:%S")
end_time1 = time.strptime(end_time, "%Y-%m-%d %H:%M:%S")
start_time2 = int(time.mktime(start_time1))
end_time2 = int(time.mktime(end_time1))
cost_sec = end_time2 - start_time2
print(str(end_time) + " " + threading.current_thread().getName() + " 第" + str(
num) + "次创建文件结束 , 花费 " + str(cost_sec) + "秒 success finished")
# print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
time.sleep(30- cost_sec)
num += 1
except Exception as e:
print("ERROR!!!")
# time_flag = str(time.time())[5:10]
# for e_path in file_path_list:
# for i in range(int(file_num)):
# path_new_file = os.path.join(e_path, "testfile_"+str(time_flag) +"_"+ str(i))
# shutil.copyfile('./demo.txt', path_new_file)
#定制线程类
class Mythread_qywu(threading.Thread):
def __init__(self, filedir_list, eve_file_num, thname):
"""
初始化方法
:param filedir_list: 所有子文件夹
:param eve_file_num: 每个文件夹下待创建文件个数
:param thname: 线程名
"""
threading.Thread.__init__(self)
self.filedir_list = filedir_list
self.eve_file_num = eve_file_num
self.thname = thname
def getName(self):
return self.thname
def run(self):
##线程调用方法
create_file(self.filedir_list, self.eve_file_num)
def avg_l(filepath_count, thread_num):
"""
根据计划线程数 均分文件夹list
:param filepath_count:
:param thread_num:
:return: avg_list [[]..]
"""
# for i in range(0, len(filepath_count), thread_num):
# yield filepath_count[i:i + thread_num]
# lists = [i for i in range(6)]
length = len(filepath_count)
n = thread_num
avg_list = []
for i in range(n):
one_list = filepath_count[math.floor(i / n * length):math.floor((i + 1) / n * length)]
avg_list.append(one_list)
return avg_list
if __name__ == "__main__":
# 文件创建的主目录
ftp_root_dir = "./res_file_dir"
#创建的子文件夹个数
filepath_count = 5
#每个子文件夹下的文件数
eve_file_num = 5
#启动的创建线程数
thread_num = 2
##储存子文件夹路劲的list
filepath_list = []
#创建子文件夹模块
for i in range(int(filepath_count)):
dir_name = ftp_root_dir + '/' + "test_dir_" + str(i)
if os.path.exists(dir_name):
pass
else:
os.mkdir(dir_name)
filepath_list.append(dir_name)
test_path_list = []
##根据文件夹路径list和计划线程数,均分任务
res_list = avg_l(filepath_list, thread_num)
thread_list = []
##生成thread对象的list
for i in range(len(res_list)):
my_thread = Mythread_qywu(res_list[i], eve_file_num, "file_create_thread_" + str(i))
thread_list.append(my_thread)
##创建线程启动
for e_thread in thread_list:
e_thread.start()
程序依赖一个作为赋值源的文件“./demo.txt”可以自由生成
执行情况如下: