# -*- coding: utf-8 -*- import datetime, time, json, re, os #from pwd import getpwnam #quality str_quality = ''.join(['{StartTime}|||{SubscriberID}', \ '|'*11, '{Bitrate}|{MOSAvg}|{DF}|{MLR}||||', '{CPUUsageHistogram}|', \ '{ProfilePlayoutSecondsHistogram_M}', '|'*8, '{ProfilePlayoutSecondsHistogram_m}', \ '|'*8, '{PlayDuration_M}|{BadDuration_M}|{PlayDuration_S}|{BadDuration_S}||||', \ '{CurrentPlaySuccNum_CurrentPlayErrNum}|{CurrentPlaySuccNum}', '|'*11, \ '{TVGuideReqNum}|{TVGuideSuccessNum}|{TVGuideDelayAvg}|||||', \ '{StallingCount_S}|{StallingDuration_S}||', \ '{BufferTrack_A}|{BufferTrack_M}|{BufferTrack_m}|||', \ '{RTTTrack_A}|{RTTTrack_M}|{RTTTrack_m}|{LostRatioTrack_A}|{LostRatioTrack_M}|{LostRatioTrack_m}', \ '|'*7, '{RAMUsageHistogram_A}|{RAMUsageHistogram_M}|{RAMUsageHistogram_m}|{DiskUsage}', \ '|'*6, '{DownloadSpeed_A}|{DownloadSpeed_M}|{DownloadSpeed_m}||||', \ '{UpBandWidthHistogram_A}|{UpBandWidthHistogram_M}|{UpBandWidthHistogram_m}|', \ '{DownBandWidthHistogram_A}|{DownBandWidthHistogram_M}|{DownBandWidthHistogram_m}||', \ '{URL}|{ServiceType}|{URL_IP}||||', '{DeviceSupplier}||||\n']) #behavior str_behavior = ''.join(['{StartTime}|{SubscriberID}', \ '|'*7, '{ServiceType}', '|'*9, '{PlayDuration_M}', \ '|'*19, '{URL}|{URL_IP}||||\n']) #CPU, RAM c_r = [0.1, 0.35, 0.6, 0.75, 0.85, 0.95] #up up = [125, 375, 750, 1500, 3000, 5000] #down, ProfilePlayoutSecondsHistogram down = [1000, 3500, 7500, 15000, 35000, 60000] pattern = re.compile(r'http://(\d+.\d+.\d+.\d+)/.*') #get the log file path def get_log_path(path): for fpathe,dirs,fs in os.walk(path): for f in fs: if f.endswith('.log'): return os.path.join(fpathe, f) def get_his_MIN(lis): for x in lis: if x != 0: return lis.index(x) return 0 def get_his_MAX(lis): for x in lis[::-1]: if x != 0: return lis.index(x) return 0 #CPU, RAM, up down def get_his_AVE_R_C(lis): s = int(100*(0.1*lis[0]+0.35*lis[1]+0.6*lis[2]+0.75*lis[3]+0.85*lis[4]+0.95*lis[5])) if sum(lis) != 0: return int (s/ sum(lis)) else: return 0 def get_up_AVE(lis): s = int(125*lis[0]+375*lis[1]+750*lis[2]+1500*lis[3]+3000*lis[4]+5000*lis[5]) if sum(lis) != 0: return int(s / sum(lis)) else: return 0 def get_down_AVE(lis): s = int(1000*lis[0]+3500*lis[1]+7500*lis[2]+15000*lis[3]+35000*lis[4]+60000*lis[5]) if sum(lis) != 0: return int(s / sum(lis)) else: return 0 #get max PlayDuration time < 300 def get_max_PlayDuration(lis_b_p): lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300] if not lis_b_p_: return 300 else: return max(lis_b_p_, key=lambda x:x[0])[1] def get_sum_PlayDuration(lis_p): lis_p_ = [x for x in lis_p if x <= 300] if sum(lis_p_) <= 300: return sum(lis_p_) return 300 #get the max badduration value，make sure the value is smaller than playduration def get_max_BadDuration(lis_b_p): lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300] if not lis_b_p_: return 0 else: return max(lis_b_p_, key=lambda x:x[0])[0] def get_sum_BadDuration(lis_b, sum_p): lis_b_ = [x for x in lis_b if x <= 300] if sum(lis_b_) < sum_p: return sum(lis_b_) else: return sum_p #recursively get all the dict of a json line def get_outcome_iter(dict_): for key, value in dict_.items(): if isinstance(value, dict): yield from get_outcome_iter(value) elif isinstance(value, list): for v in value: yield from get_outcome_iter(v) else: yield key, value def get_dict_str(json_2_dict): str_quality_tmp, str_behavior_tmp = str_quality, str_behavior str_quality_dict = dict(StartTime='', SubscriberID='', Bitrate='', MOSAvg='', DF='', MLR='', CPUUsageHistogram='', ProfilePlayoutSecondsHistogram_M='', ProfilePlayoutSecondsHistogram_m='', PlayDuration_M='', BadDuration_M='', PlayDuration_S='', BadDuration_S='', CurrentPlaySuccNum_CurrentPlayErrNum='', CurrentPlaySuccNum='', TVGuideReqNum='', TVGuideSuccessNum='', TVGuideDelayAvg='', StallingCount_S='', StallingDuration_S='', BufferTrack_A='', BufferTrack_M='', BufferTrack_m='', RTTTrack_A='', RTTTrack_M='', RTTTrack_m='', LostRatioTrack_A='', LostRatioTrack_M='', LostRatioTrack_m='', RAMUsageHistogram_A='', RAMUsageHistogram_M='', RAMUsageHistogram_m='', DiskUsage='', DownloadSpeed_A='', DownloadSpeed_M='', DownloadSpeed_m='', UpBandWidthHistogram_A='', UpBandWidthHistogram_M='', UpBandWidthHistogram_m='', DownBandWidthHistogram_A='', DownBandWidthHistogram_M='', DownBandWidthHistogram_m='', URL='', ServiceType='', URL_IP='', DeviceSupplier='') str_behavior_dict = dict(StartTime='', SubscriberID='',ServiceType='', PlayDuration_M='', URL='', URL_IP='') str_dict_tmp = dict(StartTime=[], SubscriberID=[], Bitrate=[], MOSAvg=[], DF=[], MLR=[], CPUUsageHistogram=[], ProfilePlayoutSecondsHistogram=[], PlayDuration=[], BadDuration=[], CurrentPlaySuccNum=[], CurrentPlayErrNum=[], TVGuideReqNum=[], TVGuideSuccessNum=[], TVGuideDelayAvg=[], StallingCount=[], StallingDuration=[], BufferTrack=[], RTTTrack=[], LostRatioTrack=[], RAMUsageHistogram=[], DiskUsage=[], DownloadSpeed=[], UpBandWidthHistogram=[], DownBandWidthHistogram=[], URL=[], ServiceType=[], DeviceSupplier=[]) Histogram_list = ['CPUUsageHistogram', 'RAMUsageHistogram', 'UpBandWidthHistogram', 'ProfilePlayoutSecondsHistogram', 'DownBandWidthHistogram', 'RTTTrack', 'LostRatioTrack', 'BufferTrack'] line_dict = get_outcome_iter(json_2_dict) for key, value in line_dict: if value == None or value == '': continue if key in Histogram_list: value = [int(x) for x in value.split(',')] if key in str_dict_tmp: str_dict_tmp[key].append(value) if str_dict_tmp['StartTime']: str_quality_dict['StartTime'] = str_dict_tmp['StartTime'][0] str_behavior_dict['StartTime'] = str_dict_tmp['StartTime'][0] if str_dict_tmp['SubscriberID']: str_quality_dict['SubscriberID'] = str_dict_tmp['SubscriberID'][0] str_behavior_dict['SubscriberID'] = str_dict_tmp['SubscriberID'][0] if str_dict_tmp['Bitrate']: str_quality_dict['Bitrate'] = round(max(str_dict_tmp['Bitrate']) / 1024, 2) if str_dict_tmp['MOSAvg']: str_quality_dict['MOSAvg'] = round(str_dict_tmp['MOSAvg'][0] / 10, 1) if str_dict_tmp['DF']: str_quality_dict['DF'] = int(str_dict_tmp['DF'][0] / 1000) if str_dict_tmp['MLR']: str_quality_dict['MLR'] = round(str_dict_tmp['MLR'][0] / 1000000, 1) if str_dict_tmp['CPUUsageHistogram']: str_quality_dict['CPUUsageHistogram'] = get_his_AVE_R_C(str_dict_tmp['CPUUsageHistogram'][0]) if str_dict_tmp['ProfilePlayoutSecondsHistogram']: str_quality_dict['ProfilePlayoutSecondsHistogram_M'] = \ down[get_his_MAX(str_dict_tmp['ProfilePlayoutSecondsHistogram'][0])] str_quality_dict['ProfilePlayoutSecondsHistogram_m'] = \ down[get_his_MIN(str_dict_tmp['ProfilePlayoutSecondsHistogram'][0])] if str_dict_tmp['PlayDuration'] and str_dict_tmp['BadDuration']: lis_b_p = list(zip(str_dict_tmp['BadDuration'], str_dict_tmp['PlayDuration'])) str_quality_dict['PlayDuration_M'] = get_max_PlayDuration(lis_b_p) str_quality_dict['PlayDuration_S'] = get_sum_PlayDuration(str_dict_tmp['PlayDuration']) str_behavior_dict['PlayDuration_M'] = str_quality_dict['PlayDuration_M'] str_quality_dict['BadDuration_M'] = get_max_BadDuration(lis_b_p) str_quality_dict['BadDuration_S'] = \ get_sum_BadDuration(str_dict_tmp['BadDuration'], str_quality_dict['PlayDuration_S']) if str_dict_tmp['CurrentPlaySuccNum'] and str_dict_tmp['CurrentPlayErrNum']: str_quality_dict['CurrentPlaySuccNum_CurrentPlayErrNum'] = \ str_dict_tmp['CurrentPlaySuccNum'][0] + str_dict_tmp['CurrentPlayErrNum'][0] str_quality_dict['CurrentPlaySuccNum'] = str_dict_tmp['CurrentPlaySuccNum'][0] if str_dict_tmp['TVGuideReqNum']: str_quality_dict['TVGuideReqNum'] = str_dict_tmp['TVGuideReqNum'][0] if str_dict_tmp['TVGuideSuccessNum']: str_quality_dict['TVGuideSuccessNum'] = str_dict_tmp['TVGuideSuccessNum'][0] if str_dict_tmp['TVGuideDelayAvg']: str_quality_dict['TVGuideDelayAvg'] = str_dict_tmp['TVGuideDelayAvg'][0] if str_dict_tmp['StallingDuration']: str_quality_dict['StallingDuration_S'] = sum(str_dict_tmp['StallingDuration']) if str_dict_tmp['StallingCount']: str_quality_dict['StallingCount_S'] = sum(str_dict_tmp['StallingCount']) if str_dict_tmp['BufferTrack']: str_quality_dict['BufferTrack_A'] = sum(str_dict_tmp['BufferTrack'][0]) str_quality_dict['BufferTrack_M'] = max(str_dict_tmp['BufferTrack'][0]) str_quality_dict['BufferTrack_m'] = min(str_dict_tmp['BufferTrack'][0]) if str_dict_tmp['RTTTrack']: str_quality_dict['RTTTrack_A'] = sum(str_dict_tmp['RTTTrack'][0]) str_quality_dict['RTTTrack_M'] = max(str_dict_tmp['RTTTrack'][0]) str_quality_dict['RTTTrack_m'] = min(str_dict_tmp['RTTTrack'][0]) if str_dict_tmp['LostRatioTrack']: str_quality_dict['LostRatioTrack_A'] = \ sum(str_dict_tmp['LostRatioTrack'][0]) / (1000000 * len(str_dict_tmp['LostRatioTrack'][0])) str_quality_dict['LostRatioTrack_M'] = max(str_dict_tmp['LostRatioTrack'][0]) / 1000000 str_quality_dict['LostRatioTrack_m'] = min(str_dict_tmp['LostRatioTrack'][0]) / 1000000 if str_dict_tmp['RAMUsageHistogram']: str_quality_dict['RAMUsageHistogram_A'] = get_his_AVE_R_C(str_dict_tmp['RAMUsageHistogram'][0]) str_quality_dict['RAMUsageHistogram_M'] = int(c_r[get_his_MAX(str_dict_tmp['RAMUsageHistogram'][0])] * 100) str_quality_dict['RAMUsageHistogram_m'] = int(c_r[get_his_MIN(str_dict_tmp['RAMUsageHistogram'][0])] * 100) if str_dict_tmp['DiskUsage']: str_quality_dict['DiskUsage'] = str_dict_tmp['DiskUsage'][0] if str_dict_tmp['DownloadSpeed']: str_quality_dict['DownloadSpeed_A'] = sum(str_dict_tmp['DownloadSpeed']) / len(str_dict_tmp['DownloadSpeed']) str_quality_dict['DownloadSpeed_M'] = max(str_dict_tmp['DownloadSpeed']) str_quality_dict['DownloadSpeed_m'] = min(str_dict_tmp['DownloadSpeed']) if str_dict_tmp['UpBandWidthHistogram']: str_quality_dict['UpBandWidthHistogram_A'] = get_up_AVE(str_dict_tmp['UpBandWidthHistogram'][0]) str_quality_dict['UpBandWidthHistogram_M'] = up[get_his_MAX(str_dict_tmp['UpBandWidthHistogram'][0])] str_quality_dict['UpBandWidthHistogram_m'] = up[get_his_MIN(str_dict_tmp['UpBandWidthHistogram'][0])] if str_dict_tmp['DownBandWidthHistogram']: str_quality_dict['DownBandWidthHistogram_A'] = get_down_AVE(str_dict_tmp['DownBandWidthHistogram'][0]) str_quality_dict['DownBandWidthHistogram_M'] = down[get_his_MAX(str_dict_tmp['DownBandWidthHistogram'][0])] str_quality_dict['DownBandWidthHistogram_m'] = down[get_his_MIN(str_dict_tmp['DownBandWidthHistogram'][0])] if str_dict_tmp['URL']: URL_index = str_dict_tmp['PlayDuration'].index(str_quality_dict['PlayDuration_M']) \ if str_quality_dict['PlayDuration_M'] != 300 else 0 str_quality_dict['URL'] = str_dict_tmp['URL'][URL_index] str_behavior_dict['URL'] = str_quality_dict['URL'] str_quality_dict['URL_IP'] = pattern.search(str_quality_dict['URL']).group(1) str_behavior_dict['URL_IP'] = str_quality_dict['URL_IP'] if str_dict_tmp['ServiceType']: if str_dict_tmp['ServiceType'][0] == 'LiveTV': str_dict_tmp['ServiceType'][0] = 'BTV' if str_dict_tmp['ServiceType'][0] != 'BTV' and str_dict_tmp['ServiceType'][0] != 'VOD': str_dict_tmp['ServiceType'][0] = 'PLAYBACK' str_quality_dict['ServiceType'] = str_dict_tmp['ServiceType'][0] str_behavior_dict['ServiceType'] = str_quality_dict['ServiceType'] if str_dict_tmp['DeviceSupplier']: str_quality_dict['DeviceSupplier'] = str_dict_tmp['DeviceSupplier'][0] str_quality_tmp = str_quality_tmp.format(**str_quality_dict) str_behavior_tmp = str_behavior_tmp.format(**str_behavior_dict) return str_quality_tmp, str_behavior_tmp def get_outcome_files(path_file_source, path_outcome): if path_file_source != None: ftime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") path_file_outcome = path_outcome + 'isa_stb_quality_' + ftime + '.dat' with open(path_file_outcome, 'a') as f_quality, \ open(path_file_source, 'r') as f: for line in f: if line == '<==SPLIT==>\n': continue json_2_dict = json.loads(line) outcome = get_dict_str(json_2_dict) f_quality.write(outcome[0]) #appserver_uid = getpwnam('appserver')[2] #appserver_gid = getpwnam('appserver')[3] #os.chown(path_file_outcome, appserver_uid, appserver_gid) #os.remove(path_file_source)#delete log files if __name__ == '__main__': #path_source = '/home/omc/'#folder contains log file #path_outcome = '/srv/smartcare/share/data/fbb/src/vcem/xDR'#folder contains dat file path_source = 'C:\\Users\\l00429182\\Desktop\\' path_outcome = 'C:\\Users\\l00429182\\Desktop\\' path_file_source = get_log_path(path_source)#log files that found in 5 mins print(path_file_source) get_outcome_files(path_file_source, path_outcome)
3974
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
