用python完成分析log的工作
用python分析log，ip库取自emule的ip2country库 ，
分析的内容很简单，效率大约可以做到每百万数据80s左右。
需要注意ip库是一次性加载进内存的，这样就可以节约访问数据库的时间。
查找ip对应的国家的时候用折半查找法（ip加载进内存的时候已经排好序了）。
呵呵，这种处理思路是跟caoz学的。
…………………………………………………
以下是代码：
#!d:/os/python24
# -*- coding: UTF-8 -*-
'''
This program read a log file , and find the country of every ip ,
then write the result to a output file.
Written by yuzebin : yuzebin AT gmail.com
'''
history = '''
0.0.0.2 2007-1-28
release to the internet
0.0.0.1 2007-1-15
the first version
'''
usage = '''
usage:
any_log_by_country input_log_file output_result_file
'''
import os,sys,re,locale
from pysqlite2 import dbapi2 as sqlite
from os.path import walk,join,normpath
from time import time
ip10 = {} #the begin of ip range
ip20 = {} #the end of ip range
country = {} #the country dictionary
stime=time() #the start time of this program
encoding = locale.getdefaultlocale()[1] #get the locale encoding
_swap2 = lambda (x,y): (y,x) #the swap function
def sortbyvalue(dict):
'''Return a list of (key, value) pairs, sorted by value.'''
mdict = map(_swap2, dict.items())
mdict.sort()
mdict = map(_swap2, mdict)
return mdict
def ip2int(ip):
'''Convert ip(split with ".") to integer numeric.'''
m = re.match('^(/d+)/.(/d+)/.(/d+)/.(/d+)$',ip)
if not m:
return 0
else:
return int(m.group(1))*256*256*256+int(m.group(2))*256*256+int(m.group(3))*256+int(m.group(4));
class SqliteConnection:
'''Sqlite connection manager class'''
def __init__(self,dbname):
'''Initial database connection.'''
self.conn = sqlite.connect(dbname)
self.conn.row_factory = sqlite.Row #Use field name to get the data
self.conn.text_factory=str #Fix "Could not decode to UTF-8 column"
def execute(self,sql):
'''Execute a sql statement and return the rows.'''
self.cu=self.conn.cursor()
self.cu.execute(sql)
self.conn.commit()
self.rows=[]
for self.row in self.cu:
self.rows.append(self.row)
return self.rows
def __del__(self):
'''Close the connection'''
self.conn.close()
def load_data():
'''open the ip database and loading all ip data'''
sc = SqliteConnection("ip2country.db")
sc_sql = ''
sc_row = ''
sc_sql = "select * from ip2country order by ip10"
sc_row = sc.execute(sc_sql)
i = 0
for row in sc_row:
ip10 = row[0]
ip20 = row[1]
country = row[2]
i += 1
print str(i) + ' load ok . ' + str(time()-stime) + 's'
def get_country_id(ip):
'''Find the match value in sorted table R[1..n],return 0 if no match.'''
low = 0
high = 65289 #high is the max record number.
mid = 0
while low<=high:
mid=(low+high)/2
if ip10[mid] > ip :
high = mid - 1 #Continue find in R[low..mid-1]
else:
if int(ip20[mid])<int(ip) :
low = mid + 1 #Continue find in R[mid+1..high]
else:
return mid # Find success,return the index value.
return 0
def ip2country():
print 'start loading ...'
sum = 0
d = {}
try:
fpath = sys.argv[1] #the first argv is log file
opath = sys.argv[2] #the second argv is output file
except:
print usage
return 0
#loading ip data.
load_data()
print 'start match ip .'
if os.path.isfile(fpath):
fh = open(fpath,'r')
for line in fh:
sum += 1
lines=line.split(' ')
ip = str(ip2int(lines[0]))
c_id = get_country_id(ip)
try:
try:
d[country[c_id]] += 1
except:
d[country[c_id]] = 1
except:
try:
d['other'] += 1
except:
d['other'] =1
fh.close()
sd=sortbyvalue(d)
fl = open(opath,'w')
for lt in sd:
fl.write(lt[0] + ' , ' + str(lt[1]) + '/n')
fl.close
print 'all time -> ' + str(time()-stime)
print 'all line -> ' + str(sum)
def test():
sc = SqliteConnection("ip2country.db")
sc_sql = ""
sc_row = ''
ip = '3728668051'
sc_sql = "select * from ip2country where ip10<='"
sc_sql += ip + "' and ip20>'" + ip + "' order by ip20"
temp = sc.execute(sc_sql)
print temp[0][2].decode(encoding)
if __name__ == "__main__":
ip2country()
…………………………………………………
用python完成分析log的工作
| 用python分析log，ip库取自emule的ip2country库 ， 分析的内容很简单，效率大约可以做到每百万数据80s左右。 需要注意ip库是一次性加载进内存的，这样就可以节约访问数据库的时间。 查找ip对应的国家的时候用折半查找法（ip加载进内存的时候已经排好序了）。 呵呵，这种处理思路是跟caoz学的。 ………………………………………………… 以下是代码： #!d:/os/python24 # -*- coding: UTF-8 -*- ''' This program read a log file , and find the country of every ip , then write the result to a output file. Written by yuzebin : yuzebin AT gmail.com ''' history = ''' 0.0.0.2 2007-1-28 release to the internet 0.0.0.1 2007-1-15 the first version ''' usage = ''' usage: any_log_by_country input_log_file output_result_file ''' import os,sys,re,locale from pysqlite2 import dbapi2 as sqlite from os.path import walk,join,normpath from time import time ip10 = {} #the begin of ip range ip20 = {} #the end of ip range country = {} #the country dictionary stime=time() #the start time of this program encoding = locale.getdefaultlocale()[1] #get the locale encoding _swap2 = lambda (x,y): (y,x) #the swap function def sortbyvalue(dict): '''Return a list of (key, value) pairs, sorted by value.''' mdict = map(_swap2, dict.items()) mdict.sort() mdict = map(_swap2, mdict) return mdict def ip2int(ip): '''Convert ip(split with ".") to integer numeric.''' m = re.match('^(/d+)/.(/d+)/.(/d+)/.(/d+)$',ip) if not m: return 0 else: return int(m.group(1))*256*256*256+int(m.group(2))*256*256+int(m.group(3))*256+int(m.group(4)); class SqliteConnection: '''Sqlite connection manager class''' def __init__(self,dbname): '''Initial database connection.''' self.conn = sqlite.connect(dbname) self.conn.row_factory = sqlite.Row #Use field name to get the data self.conn.text_factory=str #Fix "Could not decode to UTF-8 column" def execute(self,sql): '''Execute a sql statement and return the rows.''' self.cu=self.conn.cursor() self.cu.execute(sql) self.conn.commit() self.rows=[] for self.row in self.cu: self.rows.append(self.row) return self.rows def __del__(self): '''Close the connection''' self.conn.close() def load_data(): '''open the ip database and loading all ip data''' sc = SqliteConnection("ip2country.db") sc_sql = '' sc_row = '' sc_sql = "select * from ip2country order by ip10" sc_row = sc.execute(sc_sql) i = 0 for row in sc_row: ip10 = row[0] ip20 = row[1] country = row[2] i += 1 print str(i) + ' load ok . ' + str(time()-stime) + 's' def get_country_id(ip): '''Find the match value in sorted table R[1..n],return 0 if no match.''' low = 0 high = 65289 #high is the max record number. mid = 0 while low<=high: mid=(low+high)/2 if ip10[mid] > ip : high = mid - 1 #Continue find in R[low..mid-1] else: if int(ip20[mid])<int(ip) : low = mid + 1 #Continue find in R[mid+1..high] else: return mid # Find success,return the index value. return 0 def ip2country(): print 'start loading ...' sum = 0 d = {} try: fpath = sys.argv[1] #the first argv is log file opath = sys.argv[2] #the second argv is output file except: print usage return 0 #loading ip data. load_data() print 'start match ip .' if os.path.isfile(fpath): fh = open(fpath,'r') for line in fh: sum += 1 lines=line.split(' ') ip = str(ip2int(lines[0])) c_id = get_country_id(ip) try: try: d[country[c_id]] += 1 except: d[country[c_id]] = 1 except: try: d['other'] += 1 except: d['other'] =1 fh.close() sd=sortbyvalue(d) fl = open(opath,'w') for lt in sd: fl.write(lt[0] + ' , ' + str(lt[1]) + '/n') fl.close print 'all time -> ' + str(time()-stime) print 'all line -> ' + str(sum) def test(): sc = SqliteConnection("ip2country.db") sc_sql = "" sc_row = '' ip = '3728668051' sc_sql = "select * from ip2country where ip10<='" sc_sql += ip + "' and ip20>'" + ip + "' order by ip20" temp = sc.execute(sc_sql) print temp[0][2].decode(encoding) if __name__ == "__main__": ip2country() ………………………………………………… |
Python 在目录中查找指定的字符串
| 在目录中查找指定的字符串
|
扫一扫
2310
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
