Python数据处理并可视化

import re
import os
import sys
import time
import numpy as np
import argparse
import matplotlib.pyplot as plt
from ipywidgets import *
%matplotlib inline

'''
安装依赖：pip install jupyter
1、根目录文件下新建log文件夹（根目录文件夹路径可随意）；
2、将desktop生成的.txt日志文件重定向到log文件夹中，日志可追溯；
3、根目录下运行此脚本，在终端敲入jupyter notebook（可支持 windos，linux，mac）
4、在弹出的网页中点击get_log.ipnb,单步或整体运行脚本
'''

current_log = 'test.txt' #写入当前重定向的日志名称！
search_index = 0     #默认绘制第1个关键字的 散点和曲线图
#avatar
#searchData = ['avatar_process_texture: ','human_action_detect: ', 'frame_process: ']
#Humanaction   
#searchData = ['st_mobile_human_action_detect: ','total_process: ','fps: ']
searchData = ['human_action_detect face 1 body 0 hand 0 time: ']
# searchData = ['Model_Run_PartAlign_TIME: ']

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--log', type=str, default= current_log,help="写入当前重定向的日志名称！")
    args = parser.parse_args(args=['--log',  current_log])   #在jupyter加上此句，否则报错
    return args

def textParse(text, key_word):
    pattern = key_word + '.*?(\d+.*\d+)\.*'
    time_cost = re.findall(pattern, text)
    return time_cost

def caculate(*args):
    start, end,value = args
    mean, mid = float(np.average(value[start:end])), float(np.median(value[start:end]))
    return mean, mid

def generate_data(start, end, text):
    for i, keyword in enumerate(searchData):
        time_cost =  textParse(text, key_word)
        value =  list(map(float,time_cost ))
        num = len(time_cost)
        x = np.arange(num)
        mean, mid = caculate(start, end,value)
        print('\033[1;38m"%s" Mean: %f Mid: %f' % (keyword, mean, mid))
        if i == search_index:
            X, Y = x, value
    return X,Y
   
    
def plotTimeCost(start,end,X,Y):
    mean,  mid = caculate(start, end,Y)
    plt.figure(figsize=(10,8))
    plt.subplot(2,1,1)
    plt.ylabel('cost(ms)')
    plt.title('cost - scatter')
    plt.scatter(X[start:end],Y[start:end],c='r',marker='^',linewidths=0.5)

    figMax_x, figMax_y  = float(plt.axis()[1]),float(plt.axis()[3])
    plt.text(figMax_x+10, figMax_y - 2, "Mean: %.3f" % (mean), fontdict={'size':'12','color':'#556B2F'})
    plt.text(figMax_x+10, figMax_y - 5, "Mid: %.3f" % (mid), fontdict={'size':'12','color':'#B8860B'})
    plt.subplot(2,1,2)
    plt.xlabel('num')
    plt.ylabel('cost(ms)')
    plt.title('cost - curve')
    plt.plot(X[start:end],Y[start:end])

#    plt.savefig('scatterMemery_plot.png',dpi=500,bbox_inches = 'tight') #可以实时保存，log图片
    plt.show()
#     print('\033[1;36m均值(Mean): %f' % (mean))
#     print('\033[1;33m中值(mid-vable): %f' % (mid))

if __name__ == "__main__":
    args = parse_args()
    root_path = os.path.join(sys.path[0],'log')
    for root, dirs, files in os.walk(root_path):  
        log_list = [os.path.join(root_path, f) for f in files if f.endswith('.txt')]
    print("\033[1;36m共搜索到%d 个日志文件" % (len(log_list)))
    print("\033[1;33mfile_list: {}".format(files))
    
    key_word = searchData[search_index]   #定义所要绘制散点图的关键字
    log = os.path.join(root_path, args.log)
    if log in log_list:
        with open(log,'r', encoding='utf-8') as f:
            text = f.read()
            time_cost =  textParse(text, key_word)  #得到样本个数num
            num = len(time_cost)
            @interact(start=(0, num, 1),end=(0,num,1),continuous_update=False)
            def visualize_calculate(start=0,end=num):
                X, Y  = generate_data(start,end,text)
                plotTimeCost(start,end,X,Y)
                
    else:
         print("日志列表中不存在目标日志，请重新生成！")