python读取log文件_python3 log文件处理获取某天需要的数据

问题描述：需要读取log文件，并获取出两天的日志做对比，查询增加或者减少量，规则化输出

完成心得：python的字典，很好用，读写时间快，用法自由#!/usr/bin/env python

# -*- coding: utf-8 -*-

# @Time : 17/8/18 下午5:08

# @Author : lee

# @File : read.py

# @Software: PyCharm

# 说明: code后有'#'的时测试时加的或者需要修改的code

# 用法 传入两个值 day1是前一天的日期 day2是后一天的内容

import sys

def log_data(address, day1, day2):

try:

# 打开文件

log = open("%s" % address)

# 定义：

dict_yesterday = {} # 昨天的数据

dict_the_day_before_yesterday = {} # 前天的数据

dict_yesterday_difference = {} # 记录昨天中有的数据，前天没有

dict_the_day_before_yesterday_difference = {} # 和上一条相反

dict_identical = {} # 记录两天内相同的数据

dict_result = {} # 记录结果，输出时读取

ii = jj = z = x = 0

#遍历 将两个日期的日志筛选出并装入对应的字典

for line in log:

log_data = ','.join(filter(lambda x: x,line.split(' ')))

date = (log_data.split(',')[0])[-10:].replace('.','-')

if date == day2:

dict_yesterday[ii] = log_data

dict_yesterday_difference[ii] = log_data

ii += 1

elif date == day1:

dict_the_day_before_yesterday[jj] = log_data

dict_the_day_before_yesterday_difference[jj] = log_data

jj += 1

if len(dict_yesterday) >= len(dict_the_day_before_yesterday):

for i in dict_yesterday:

object_list1 = (dict_yesterday[i]).split(',')

entry_name_yesterday = ((object_list1[0]).split(',')[0])[:-10]+object_list1[1]+'-'+object_list1[2]

for j in dict_the_day_before_yesterday:

object_list2 = (dict_the_day_before_yesterday[j]).split(',')

entry_name_tdat = ((object_list2[0]).split(',')[0])[:-10]+object_list2[1]+'-'+object_list2[2]

if entry_name_yesterday == entry_name_tdat:

dict_identical[x] = entry_name_yesterday

docs_result = int(((dict_yesterday[i]).split(','))[4]) - int(((dict_the_day_before_yesterday[j]).split(','))[4])

capacity1 = (dict_yesterday[i].split(',')[5])[-2:]

capacity2 = (dict_the_day_before_yesterday[j].split(',')[5])[-2:]

if capacity1 == capacity2:

store_result = str(round(float((dict_yesterday[i].split(',')[5])[:-2]) - float((dict_the_day_before_yesterday[j].split(',')[5])[:-2]),2)) +(dict_yesterday[i].split(',')[5])[-2:]

# 1gb = 1024mb = 1024kb

elif capacity1 != capacity2:

n_y = round(float((dict_yesterday[i].split(',')[5])[:-2]), 2)

n_t = float((dict_the_day_before_yesterday[j].split(',')[5])[:-2])

if capacity1 == 'gb'and capacity2 == 'mb':

if n_y * 1024 > n_t:

store_result = str(abs(round(n_y * 1024 - n_t), 2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

else:

store_result = str(round(n_y * 1024 - n_t),) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

elif capacity1 == 'gb'and capacity2 == 'kb':

if n_y * 1024 * 1024 > n_t:

# round(,2)

store_result = str(abs(round(n_y * 1024 * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

else:

store_result = str(round(n_y * 1024 * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

elif capacity1 == 'mb'and capacity2 == 'gb':

if n_y / 1024 > n_t:

store_result = str(abs(round(n_y / 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

else:

store_result = str(round(n_y / 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

elif capacity1 == 'mb'and capacity2 == 'kb':

if n_y * 1024 > n_t:

store_result = str(abs(round(n_y * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

else:

store_result = str(round(n_y * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]

elif capacity1 == 'kb'and capacity2 == 'gb':

store_result = str(round(n_t - n_t * 1024 * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]

elif capacity1 == 'kb'and capacity2 == 'mb':

if n_y / 1024 > n_t:

store_result = str(abs(round(n_t - n_t * 1024,2))) +(dict_yesterday[i].split(',')[5])[-2:]

else:

store_result = str(round(n_t - n_t * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]

dict_result[z] = entry_name_tdat + ' docs: ' + str(docs_result) + ' store: ' +store_result

z += 1

x += 1

# else:

# print("当前日期下没有相关日志")

# 筛选出昨天出现 前天没有的数据

if len(dict_yesterday) >= len(dict_identical):

for i in dict_yesterday:

object_list_1 = (dict_yesterday[i]).split(',')

entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]

for j in dict_identical:

entry_name_identical = dict_identical[j]

if entry_name_yesterday_1 == entry_name_identical:

dict_yesterday_difference.pop(i)

# 筛选出前天出现 昨天没有的数据

if len(dict_the_day_before_yesterday) >= len(dict_identical):

for i in dict_the_day_before_yesterday:

object_list_1 = (dict_the_day_before_yesterday[i]).split(',')

entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]

for j in dict_identical:

entry_name_identical = dict_identical[j]

if entry_name_yesterday_1 == entry_name_identical:

dict_the_day_before_yesterday_difference.pop(i)

# 输出两天都有的数据

print("下面是 %s 和 %s 内%s中 docs、store差值"%(day1,day2,address))

print('')

print("++++++++++++++++两天都有的数据++++++++++++++++")

for i in dict_result:

print(dict_result[i])

# 输出两天中前者中有的数据而在后者没有出现，取正值(+)区别

print('')

if len(dict_yesterday) != 0 :

print("++++++++++++++++只有%s有的数据取正值(+)予以区别++++++++++++++++"%day2)

for i in dict_yesterday_difference:

object = (dict_yesterday_difference[i]).split(',')

name = ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]

docs = object[4]

store = object[5]

print(name + ' docs: +' + str(docs) + ' store: ' + store)

else:

print("%s日期下没有相关日志"%day2)

# 输出两天中后者中有的数据而在前者没有出现，取负值(-)区别

print('')

if len(dict_the_day_before_yesterday) != 0:

print("++++++++++++++++只有%s有的数据，都取负值(-)予以区别++++++++++++++++" % day1)

for i in dict_the_day_before_yesterday_difference:

object = (dict_the_day_before_yesterday_difference[i]).split(',')

name = ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]

docs = object[4]

store = object[5]

print(name+ ' docs: -' + str(docs) + ' store: -' + store)

else:

print("%s日期下没有相关日志"%day1)

except:

print("输入的参数可能不对，三个参数，地址，day1 day2")

# log_data('/Users/lee/Desktop/py交易/es.log','2017-08-11','2017-08-12')

log_data(sys.argv[1],sys.argv[2],sys.argv[3])

# /Users/lee/Desktop/py交易/test.log