因目前fio内置无批量收集测试结果的功能,如果简单的测试(测试结果100个文件以下),可以手动收集结果,但是如果大于100个log文件,处理起来会非常麻烦。
1.本文脚本运行环境:
1.1在windows X系统;
1.2.python3.4+。
2.用法:
2.1测试fio脚本输出的log必须要有‘k’ or ‘K’为名称;
2.2把此py内容拷贝到新建py文件中;
2.3将py文件拷贝到fio测试结果文件夹中;
2.4将非fio结果文件移除(建议备份);
2.5直接运行py文件即可;
2.6运行起始要点击回车继续;
2.7第一在电脑运行时会自动安装一些插件,需要等待即可
运行完毕后,文件夹中会直接将fio log 结果生成xls表格并汇总起来。
直接把下面脚本复制,并放置fio输出结果文件夹下运行即可
#!/usr/bin/env python3
#coding=utf-8
#Author:Gang.Fang
#Update Date:20190730
#Version:v2019.7
import datetime,platform,time,os,sys,stat,re
print('READ ME!!! READ ME!!! READ ME!!! \nPlease make sure your python version later than 3.4, \
put this py file into log folder and remove other logs except fio log,press "Enter" key to continue!!!')
os.system('echo READ ME!!! READ ME!!! READ ME!!! READ ME!!! READ ME!!! READ ME!!! \
Please make sure your python version later than 3.4, put this py file into log folder and remove other logs except fio log,\
press "Enter" key to continue!!! & pause')
if(os.path.exists("filedir.log")):
os.remove("filedir.log")
else:
pass
if(os.path.exists("filedir1.log")):
os.remove("filedir1.log")
else:
pass
os.system('where python >> filedir.log')
fa=open("filedir.log","r+")
dir2=str(fa.readline())
dir3=dir2.replace('python.exe','').replace('\n','')
fa.close()
if(os.path.exists(dir3+"filedir4.log")):
pass
else:
os.system('pip install urllib')
os.system('pip install requests')
os.system('pip install linecache')
os.system('pip install shutil')
os.system('pip install xlrd')
os.system('pip install xlwt')
os.system('pip install openpyxl')
os.system('pip install xlutils')
fb=open(dir3+"filedir4.log","w")
fb.close()
import linecache,requests
from urllib import request
import urllib.request
import shutil,xlrd,xlwt,openpyxl
from xlutils.copy import copy
from openpyxl import load_workbook
f0=open("filelist0.log","w")
f0.write(str(os.listdir(os.getcwd())))
f0.close()
f=open("filelist0.log","r+")
f1=open("filelist1.log","w")
line1=str(f.readline())
#显示原始文件列表
line2=line1.replace('[','').replace(']','').replace("', '",'\r\n').replace("'",'')#整理文件列表为目标列表
f1.write(line2)
f1.close()
f.close()
#去掉空白行列表
f2=open("filelist1.log","r")
f3=open("Test_result_file_name.log","w")
for line in f2:
line=line.strip()
if len(line)!=0:
if 'k' in str(line) or 'K' in str(line):
if 'bw.' not in str(line):
if 'iops.' not in str(line):
if 'lat.' not in str(line):
if '.py' not in str(line):
if '.sh' not in str(line):
if 'Pre' not in str(line):
if 'pre' not in str(line):
if 'dmesg' not in str(line):
if 'smart' not in str(line):
f3.write(str(line))
f3.write('\n')
f2.close()
f3.close()
print('Please wait for data updating')
ddate=time.strftime('%Y%m%d',time.localtime(time.time()))
#新建excel文件(带日期)
wb0=xlwt.Workbook(encoding='utf-8')
ws0=wb0.add_sheet(u"Performance_test_result")
a0=["Test_item_name","rwtype1","IOPS/k","BW/MB/s","read Lat/usec","read Max_lat/usec","rwtype2","IOPS/k","BW/MB/s","write Lat/usec","write Max_lat/usec"]
for a1 in range(len(a0)):
ws0.write(0,a1,a0[a1])
#打开并开始写入数据
str1=linecache.getlines("Test_result_file_name.log")
for i in range(len(str1)):
filename=str1[i].replace('\n','')
str3=linecache.getlines(filename)
if len(str3) > 50 or len(str3) < 20:
pass
else:
test_item=filename.split(".",1)[0]
f9=open("Temp.log","w")
ws0.write(i+1,0,test_item)
line0=str3[0].replace(' ','')
f9.write(line0)
if len(str3) < 38:
for m in range(1,len(str3)):
if 'BW' in str3[m] and 'IOPS' in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
continue
if 'clat' in str3[m] and 'percentiles' not in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
break
f9.write('0')
f9.write('\n')
f9.write('0')
else:
for m in range(1,len(str3)):
if m < 10:
if 'BW' in str3[m] and 'IOPS' in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
continue
if 'clat (' in str3[m] and 'percentiles' not in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
continue
else:
if 'BW' in str3[m] and 'IOPS' in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
continue
if 'clat (' in str3[m] and 'percentiles' not in str3[m]:
f9.write(str(str3[m]).replace(' ',''))
break
f9.close()
str4=linecache.getlines("Temp.log")
str5=str4[1]
str6=str4[2]
str7=str4[3]
str8=str4[4]
str_result1=str5.split(":",1)[0]
str_result2=str7.split(":",1)[0]
if str_result1 == "read":
if str_result2 == "write":
#第一组数据
ws0.write(i+1,1,str_result1)
#iops
str_result3=str5.split(",",1)[0]
str_result4=str_result3.split("=",1)[1]
ws0.write(i+1,2,str_result4)
#bw
str_result5=str5.split(",",1)[1]
str_result6=str_result5.split(")(",1)[0]
str_result7=str_result6.split("(",1)[1]
ws0.write(i+1,3,str_result7)
#lat
str_result8=str6.split("avg=",1)[1]
str_result9=str_result8.split(",",1)[0]
str_result10=str6.split("avg=",1)[0]
str_result11=str_result10.split("(",1)[1]
str_result12=str_result11.split(")",1)[0]
ws0.write(i+1,4,str_result9+str_result12)
#Maxlat
Maxlat1=str6.split("max=",1)[1]
Maxlat2=Maxlat1.split(",",1)[0]
Maxlat3=str6.split(":",1)[0]
Maxlat4=Maxlat3.split("(",1)[1]
Maxlat5=Maxlat4.split(")",1)[0]
ws0.write(i+1,5,Maxlat2+Maxlat5)
#第二组数据
ws0.write(i+1,6,str_result2)
#iops
str_result13=str7.split(",",1)[0]
str_result14=str_result13.split("=",1)[1]
ws0.write(i+1,7,str_result14)
#bw
str_result15=str7.split(",",1)[1]
str_result16=str_result15.split(")(",1)[0]
str_result17=str_result16.split("(",1)[1]
ws0.write(i+1,8,str_result17)
#lat
str_result18=str8.split("avg=",1)[1]
str_result19=str_result18.split(",",1)[0]
str_result20=str8.split("avg=",1)[0]
str_result21=str_result20.split("(",1)[1]
str_result22=str_result21.split(")",1)[0]
ws0.write(i+1,9,str_result19+str_result22)
#Maxlat
Maxlat6=str8.split("max=",1)[1]
Maxlat7=Maxlat6.split(",",1)[0]
Maxlat8=str8.split(":",1)[0]
Maxlat9=Maxlat8.split("(",1)[1]
Maxlat10=Maxlat9.split(")",1)[0]
ws0.write(i+1,10,Maxlat7+Maxlat10)
if(os.path.exists("Temp.log")):
os.remove("Temp.log")
else:
print("要删除的文件不存在!")
linecache.updatecache("Temp.log")
linecache.updatecache(filename)
else:
#第一组数据
#print(str_result1)
#print(str_result2)
ws0.write(i+1,1,str_result1)
#iops
str_result23=str5.split(",",1)[0]
str_result24=str_result23.split("=",1)[1]
ws0.write(i+1,2,str_result24)
#bw
str_result25=str5.split(",",1)[1]
str_result26=str_result25.split(")(",1)[0]
str_result27=str_result26.split("(",1)[1]
ws0.write(i+1,3,str_result27)
#lat
str_result28=str6.split("avg=",1)[1]
str_result29=str_result28.split(",",1)[0]
str_result30=str6.split("avg=",1)[0]
str_result31=str_result30.split("(",1)[1]
str_result32=str_result31.split(")",1)[0]
ws0.write(i+1,4,str_result29+str_result32)
#Maxlat
Maxlat11=str6.split("max=",1)[1]
Maxlat12=Maxlat11.split(",",1)[0]
Maxlat13=str6.split(":",1)[0]
Maxlat14=Maxlat13.split("(",1)[1]
Maxlat15=Maxlat14.split(")",1)[0]
ws0.write(i+1,5,Maxlat12+Maxlat15)
#第二组数据
ws0.write(i+1,6,0)
#iops
ws0.write(i+1,7,0)
#bw
ws0.write(i+1,8,0)
#lat
ws0.write(i+1,9,0)
#Maxlat
ws0.write(i+1,10,0)
if(os.path.exists("Temp.log")):
os.remove("Temp.log")
else:
print("要删除的文件不存在!")
linecache.updatecache("Temp.log")
linecache.updatecache(filename)
else:
ws0.write(i+1,1,0)
#iops
ws0.write(i+1,2,0)
#bw
ws0.write(i+1,3,0)
#lat
ws0.write(i+1,4,0)
#Maxlat
ws0.write(i+1,5,0)
#第二组数据
ws0.write(i+1,6,str_result1)
#iops
str_result33=str5.split(",",1)[0]
str_result34=str_result33.split("=",1)[1]
ws0.write(i+1,7,str_result34)
#bw
str_result35=str5.split(",",1)[1]
str_result36=str_result35.split(")(",1)[0]
str_result37=str_result36.split("(",1)[1]
ws0.write(i+1,8,str_result37)
#lat
str_result38=str6.split("avg=",1)[1]
str_result39=str_result38.split(",",1)[0]
str_result40=str6.split("avg=",1)[0]
str_result41=str_result40.split("(",1)[1]
str_result42=str_result41.split(")",1)[0]
ws0.write(i+1,9,str_result39+str_result42)
#Maxlat
Maxlat16=str6.split("max=",1)[1]
Maxlat17=Maxlat16.split(",",1)[0]
Maxlat18=str6.split(":",1)[0]
Maxlat19=Maxlat18.split("(",1)[1]
Maxlat20=Maxlat19.split(")",1)[0]
ws0.write(i+1,10,Maxlat17+Maxlat20)
linecache.updatecache("Temp.log")
linecache.updatecache(filename)
if(os.path.exists("Temp.log")):
os.remove("Temp.log")
else:
print("要删除的文件不存在!")
linecache.updatecache("Temp.log")
linecache.updatecache(filename)
wb0.save('Performance_test_result.xlsx')
wbb0=xlrd.open_workbook('Performance_test_result.xlsx')
wss0=wbb0.sheets()[0]
Nrows=wss0.nrows
Ncols=wss0.ncols
wbb=copy(wbb0)
wss=wbb.get_sheet(0)
for i in range(1,Nrows):
for m in range(1,Ncols):
if m in [0,1,6]:
n=wss0.cell(i,m).value
wss.write(i,m,n)
elif m in [2,7]:
iops=wss0.cell(i,m).value
if 'k' in str(iops):
iops=str(iops).replace('k','')
elif 'k' not in str(iops):
iops=round(int(iops)/1000,2)
wss.write(i,m,iops)
elif m in [3,8]:
bw=wss0.cell(i,m).value
if 'MB/s' in str(bw):
bw=str(bw).replace('MB/s','')
elif 'KB/s' in str(bw):
bw1=str(bw).replace('KB/s','')
bw=round(float(bw1)/1024,2)
elif 'GB/s' in str(bw):
bw1=str(bw).replace('GB/s','')
bw=round(float(bw1)*1024,2)
wss.write(i,m,bw)
else:
lat=wss0.cell(i,m).value
if 'usec' in str(lat) :
lat=str(lat).replace('usec','')
elif 'knsec' in str(lat) :
lat=str(lat).replace('knsec','')
elif 'nsec' in str(lat):
lat1=str(lat).replace('nsec','')
lat=round(float(lat1)/1000,2)
elif 'msec' in str(lat):
lat1=str(lat).replace('msec','')
lat=round(float(lat1)*1000,2)
wss.write(i,m,lat)
wbb.save('Performance_test_result1.xls')
os.remove("Performance_test_result.xlsx")
os.remove("filedir.log")
os.remove("filelist0.log")
os.remove("filelist1.log")
os.remove("Test_result_file_name.log")
参考:https://blog.csdn.net/morgan_fang1/article/details/97883973