数据提取与绘图笔记

• 删除文件中不需要的内容：

             

例子：

def deleate_unnecessary_data(file_path, deleat_str=r'=======After\sloop\s\d*\s=======\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n'):
    with open(file_path, 'r+') as file:
        content = file.read()
        new_content = re.sub(deleat_str, '', content)

        file.seek(0)
        file.truncate()
        file.write(new_content)
        file.close()

   传入 文件路径以及 不需要的内容的 正则表达式  将文件中不需要的内容删掉

• 从文件中查找需要的内容并存入文件中

例子：

def get_need_data(file_path, match_str=r'WAF\s.NAND.\s=\s\d.*'):
    with open(file_path, 'r') as file:
        content = file.read()
        matches = re.findall(match_str, content)
        with open('target_file.txt', 'w') as target:
            for match in matches:
                target.write(match + '\n')
            target.close()
        file.close()

传入 目标文件路径  以及  需要匹配的内容的  正则表达式 ，将目标文件中匹配到的内容谢入新的文件中。

• 获取目标路径下的文件名列表

file_list = os.listdir(path)

• 将文件内容读出来存入列表中

def read_file_to_list(file_path):
    with open(file_path, 'r') as file:
        content = file.read()
    return content.splitlines()

• 将数据存入dataframe 以及绘图：

import re
import os, sys
from matplotlib.ticker import MultipleLocator
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt


def deleate_unnecessary_data(file_path, deleat_str=r'=======After\sloop\s\d*\s=======\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n.*\n'):
    with open(file_path, 'r+') as file:
        content = file.read()
        new_content = re.sub(deleat_str, '', content)

        file.seek(0)
        file.truncate()
        file.write(new_content)
        file.close()


def get_need_data(file_path, match_str=r'WAF\s.NAND.\s=\s\d.*', deleat_str=r'WAF\s.NAND.\s=\s'):
    with open(file_path, 'r') as file:
        content = file.read()
        matches = re.findall(match_str, content)
        with open('target_file.txt', 'w') as target:
            for match in matches:
                target.write(match + '\n')
            target.close()
        file.close()

    deleate_unnecessary_data('target_file.txt', deleat_str=deleat_str)

    return 'target_file.txt'


def read_file_to_list(file_path):
    with open(file_path, 'r') as file:
        content = file.read()
    return content.splitlines()


device = 'aaa'
# test_mode = 'vvv'

path = "./test_log/"
file_list = os.listdir(path)
start = 5
step = 5
len_max = 10

df = pd.DataFrame(columns=[])

for file_n in file_list:
    file_path = path + file_n
    df1 = pd.DataFrame(columns=[])
    base_name, extension = os.path.splitext(os.path.basename(file_path))

    deleate_unnecessary_data(file_path)

    header = 'NAND_WAF_' + base_name
    content = read_file_to_list(get_need_data(file_path, match_str=r'WAF\s.NAND.\s=\s\d.*', deleat_str=r'WAF\s.NAND.\s=\s')) 
    df1[header] = content

    header = 'SLC_WAF_' + base_name
    content = read_file_to_list(get_need_data(file_path, match_str=r'WAF\s.SLC.\s=\s\d.*', deleat_str=r'WAF\s.SLC.\s=\s')) 
    df1[header] = content

    df = pd.concat([df, df1], axis=1)

    len_max = max(len_max, len(content))

end = (len_max+1)*step
df1 = pd.DataFrame(columns=[])
data_x ='size of data written(GB)'
df1[data_x] = np.arange(start, end, step)
df = pd.concat([df, df1], axis=1)
df.insert(0, data_x, df.pop(data_x))

df.to_csv('output.csv', index=False)


df = pd.read_csv('output.csv')
column_names_list = list(df.columns)
nand_data = [col for col in column_names_list if re.match(r'NAND_WAF.*', col)]
slc_data = [col for col in column_names_list if re.match(r'SLC_WAF.*', col)]

traindata=pd.DataFrame(df,dtype=np.float64)

plt.rcParams['font.size'] = 20

#NAND#
fig, ax = plt.subplots(figsize=(20, 10))
for i, col in enumerate(nand_data):
    ax.plot(df[data_x], df[col], label=col, linewidth=3.0)

# ax.set_title(r"$\bf{" + str("AM6A0 Nand-WAF strend") + "}$"+'\n(rand write with block_size=4K, flush afte every write)')  # 图表标题
ax.set_title(device + ' Nand-WAF strend \n(rand write with block_size=4K, '+ test_mode, fontweight='bold')  # 图表标题
ax.set_xlabel(data_x, fontweight='bold')  # X轴标签
ax.set_ylabel('WAF', fontweight='bold')  # Y轴标签
ax.legend()

# ax.yaxis.set_major_locator(MultipleLocator(10))
plt.grid()
plt.show()
f = plt.gcf()  #获取当前图像
f.savefig(r'./{}.png'.format('NAND_WAF'))
f.clear()  #释放内存


#SLC#
fig, ax = plt.subplots(figsize=(20, 10))
for i, col in enumerate(slc_data):
    ax.plot(df[data_x], df[col], label=col, linewidth=3.0)

ax.set_title(device + ' SLC-WAF trend\n(rand write with block_size=4K, '+ test_mode,fontweight='bold')  # 图表标题
ax.set_xlabel(data_x, fontweight='bold')  # X轴标签
ax.set_ylabel('WAF', fontweight='bold')  # Y轴标签
ax.legend()

# ax.yaxis.set_major_locator(MultipleLocator(2))
plt.grid()
plt.show()
f = plt.gcf()  #获取当前图像
f.savefig(r'./{}.png'.format('SLC_WAF'))
f.clear()  #释放内存