1.python的writer做法
import csv
import pandas as pd
import os
from statistics import mean
#对dev测试生成的多个csv进行融合
def merge_different_csv():
CSV_C0_NAME = 'file_name'
CSV_C1_NAME = 'real_length'
CSV_C2_NAME = 'dev_length'
CSV_C3_NAME = 'dev_length_abs_error'
CSV_C4_NAME = 'dev_length_relative_error'
CSV_C5_NAME = 'prod_length'
CSV_C6_NAME = 'prod_length_abs_error'
CSV_C7_NAME = 'prod_length_relative_error'
CSV_C8_NAME = 'real_weight'
CSV_C9_NAME = 'dev_weight'
CSV_C10_NAME = 'dev_weight_abs_error'
CSV_C11_NAME = 'dev_weight_relative_error'
CSV_C12_NAME = 'prod_weight'
CSV_C13_NAME = 'prod_weight_abs_error'
CSV_C14_NAME = 'prod_weight_relative_error'
CSV_C15_NAME = 'dev_time'
CSV_C16_NAME = 'prod_time'
csv_path='./data/csv_data'
every_csv__path_list_names=[os.path.join(csv_path,i) for i in os.listdir(csv_path)]
dev_length_error=[]
prod_length_error = []
dev_weight_error = []
prod_weight_error = []
# 计算mean
for every_csv_path in every_csv__path_list_names:
with open(every_csv_path, "r") as csvfile:
reader2 = csv.reader(csvfile) # 读取csv文件,返回的是迭代类型
# print(reader2)
for i,csv_list in enumerate(reader2):
# print(csv_list)
if i:
csv_rows ={CSV_C0_NAME: csv_list[0],
CSV_C1_NAME: csv_list[1],
CSV_C2_NAME: csv_list[2],
CSV_C3_NAME: csv_list[3],
CSV_C4_NAME: csv_list[4],
CSV_C5_NAME: csv_list[5],
CSV_C6_NAME: csv_list[6],
CSV_C7_NAME: csv_list[7],
CSV_C8_NAME: csv_list[8],
CSV_C9_NAME: csv_list[9],
CSV_C10_NAME: csv_list[10],
CSV_C11_NAME: csv_list[11],
CSV_C12_NAME: csv_list[12],
CSV_C13_NAME: csv_list[13],
CSV_C14_NAME: csv_list[14],
CSV_C15_NAME: csv_list[15],
CSV_C16_NAME: csv_list[16]}
dev_length_error.append(float(csv_rows.get(CSV_C4_NAME)))
prod_length_error.append(float(csv_rows.get(CSV_C7_NAME)))
dev_weight_error.append(float(csv_rows.get(CSV_C11_NAME)))
prod_weight_error.append(float(csv_rows.get(CSV_C14_NAME)))
# print(round(mean(dev_length_error),2))
# print(round(mean(prod_length_error), 2))
# print(round(mean(dev_weight_error), 2))
# print(round(mean(prod_weight_error), 2))
CSV_C4_NAME = CSV_C4_NAME + '=' + str(round(mean(dev_length_error), 2)) + '%'
CSV_C7_NAME = CSV_C7_NAME + '=' + str(round(mean(prod_length_error), 2)) + '%'
CSV_C11_NAME = CSV_C11_NAME + '=' + str(round(mean(dev_weight_error), 2)) + '%'
CSV_C14_NAME = CSV_C14_NAME + '=' + str(round(mean(prod_weight_error), 2)) + '%'
print(CSV_C4_NAME, CSV_C7_NAME, CSV_C11_NAME, CSV_C14_NAME)
fieldnames_l = [CSV_C0_NAME, CSV_C1_NAME, CSV_C2_NAME, CSV_C3_NAME, CSV_C4_NAME, CSV_C5_NAME, CSV_C6_NAME,
CSV_C7_NAME, CSV_C8_NAME, CSV_C9_NAME, CSV_C10_NAME, CSV_C11_NAME, CSV_C12_NAME, CSV_C13_NAME,
CSV_C14_NAME, CSV_C15_NAME, CSV_C16_NAME]
csvfile_all = open('./data/pig_dev_prod_data.csv', 'w')
writer_pig_data = csv.DictWriter(csvfile_all, fieldnames=fieldnames_l)
writer_pig_data.writeheader()
for every_csv_path in every_csv__path_list_names:
with open(every_csv_path, "r") as csvfile:
reader2 = csv.reader(csvfile) # 读取csv文件,返回的是迭代类型
# print(reader2)
for i,csv_list in enumerate(reader2):
# print(csv_list)
if i:
csv_rows ={CSV_C0_NAME: csv_list[0],
CSV_C1_NAME: csv_list[1],
CSV_C2_NAME: csv_list[2],
CSV_C3_NAME: csv_list[3],
CSV_C4_NAME: csv_list[4],
CSV_C5_NAME: csv_list[5],
CSV_C6_NAME: csv_list[6],
CSV_C7_NAME: csv_list[7],
CSV_C8_NAME: csv_list[8],
CSV_C9_NAME: csv_list[9],
CSV_C10_NAME: csv_list[10],
CSV_C11_NAME: csv_list[11],
CSV_C12_NAME: csv_list[12],
CSV_C13_NAME: csv_list[13],
CSV_C14_NAME: csv_list[14],
CSV_C15_NAME: csv_list[15],
CSV_C16_NAME: csv_list[16]}
writer_pig_data.writerow(csv_rows)
2.pandas做法
import os
import pandas as pd
#对dev测试生成的多个csv进行融合
def merge_different_csv():
"""notice"""
out_csv_path='pig_dev_prod_data.csv'
csv_path = './out_csv'
every_csv_path_list_names=[os.path.join(csv_path,i) for i in os.listdir(csv_path)]
df=pd.read_csv(every_csv_path_list_names[-1])
df.to_csv(out_csv_path, sep=',', encoding='utf-8', index=False)
for i in range(len(every_csv_path_list_names)-1):
df = pd.read_csv(every_csv_path_list_names[i])
df.to_csv(out_csv_path,encoding='utf-8',index=False,header=False, mode='a+')
df_all=pd.read_csv(out_csv_path)
dev_length_relative_error=df_all['dev_length_relative_error']
prod_length_relative_error=df_all['prod_length_relative_error']
real_length=df_all['real_length']
mean_dev_length=round(sum(dev_length_relative_error) / sum(real_length) * 100,2)
mean_prod_length=round(sum(prod_length_relative_error) / sum(real_length) * 100,2)
dev_weight_relative_error = df_all['dev_weight_relative_error']
prod_weight_relative_error = df_all['prod_weight_relative_error']
real_weight = df_all['real_weight']
mean_dev_weight=round(sum(dev_weight_relative_error)/sum(real_weight)*100,2)
mean_prod_weight=round(sum(prod_weight_relative_error) / sum(real_weight) * 100,2)
print('测试模型长度误差 %.2f, 生产模型长度误差 %.2f' % (mean_dev_length, mean_prod_length))
print('测试模型重量误差 %.2f, 生产模型重量误差 %.2f' % (mean_dev_weight, mean_prod_weight))
dev_length_error_col_name = 'dev_length_relative_error {0}%'.format(mean_dev_length)
prod_length_error_col_name = 'prod_length_relative_error {0}%'.format(mean_prod_length)
dev_weight_error_col_name = 'dev_weight_relative_error {0}%'.format(mean_dev_weight)
prod_weight_error_col_name = 'prod_weight_relative_error {0}%'.format(mean_prod_weight)
df_all.rename(columns={'dev_length_relative_error': dev_length_error_col_name,
'prod_length_relative_error': prod_length_error_col_name,
'dev_weight_relative_error': dev_weight_error_col_name,
'prod_weight_relative_error': prod_weight_error_col_name
},inplace=True)
df_all.to_csv(out_csv_path, encoding='utf-8',index=False)
if __name__ == '__main__':
merge_different_csv()