Python 在目录中查找指定的字符串 python完成分析log的工作

最新推荐文章于 2021-04-06 13:52:40 发布

kbkiss_1

最新推荐文章于 2021-04-06 13:52:40 发布

阅读量3k

点赞数

文章标签： python import sql encoding output database

用python完成分析log的工作

用python分析log，ip库取自emule的ip2country库，
分析的内容很简单，效率大约可以做到每百万数据80s左右。
需要注意ip库是一次性加载进内存的，这样就可以节约访问数据库的时间。
查找ip对应的国家的时候用折半查找法（ip加载进内存的时候已经排好序了）。
呵呵，这种处理思路是跟caoz学的。

…………………………………………………
以下是代码：
#!d:/os/python24
# -- coding: UTF-8 --
'''
This program read a log file , and find the country of every ip ,
      then write the result to a output file.
Written by yuzebin : yuzebin AT gmail.com
'''

history = '''
0.0.0.2 2007-1-28
      release to the internet

0.0.0.1 2007-1-15
      the first version
'''

usage = '''
usage:
      any_log_by_country input_log_file output_result_file
'''

import os,sys,re,locale
from pysqlite2 import dbapi2 as sqlite
from os.path import walk,join,normpath
from time import time

ip10 = {}             #the begin of ip range
ip20 = {}             #the end of ip range
country = {}       #the country dictionary
stime=time()       #the start time of this program
encoding = locale.getdefaultlocale()[1]    #get the locale encoding

_swap2 = lambda (x,y): (y,x) #the swap function
def sortbyvalue(dict):
'''Return a list of (key, value) pairs, sorted by value.'''
mdict = map(_swap2, dict.items())
mdict.sort()
mdict = map(_swap2, mdict)
return mdict

def ip2int(ip):
'''Convert ip(split with ".") to integer numeric.'''
m = re.match('^(/d+)/.(/d+)/.(/d+)/.(/d+)$',ip)
if not m:
      return 0
else:
      return int(m.group(1))256256256+int(m.group(2))256256+int(m.group(3))256+int(m.group(4));

class SqliteConnection:
'''Sqlite connection manager class'''
def init(self,dbname):
      '''Initial database connection.'''
      self.conn = sqlite.connect(dbname)
      self.conn.row_factory = sqlite.Row #Use field name to get the data
      self.conn.text_factory=str #Fix "Could not decode to UTF-8 column"

def execute(self,sql):
      '''Execute a sql statement and return the rows.'''
      self.cu=self.conn.cursor()
      self.cu.execute(sql)
      self.conn.commit()

      self.rows=[]
      for self.row in self.cu:
         self.rows.append(self.row)
      return self.rows

def del(self):
      '''Close the connection'''
      self.conn.close()

def load_data():
'''open the ip database and loading all ip data'''
sc = SqliteConnection("ip2country.db")
sc_sql = ''
sc_row = ''

sc_sql = "select * from ip2country order by ip10"
sc_row = sc.execute(sc_sql)

i = 0
for row in sc_row:
      ip10 = row[0]
      ip20 = row[1]
      country = row[2]
      i += 1
print str(i) + ' load ok . ' + str(time()-stime) + 's'

def get_country_id(ip):
'''Find the match value in sorted table R[1..n],return 0 if no match.'''
low = 0
high = 65289 #high is the max record number.
mid = 0

while low<=high:
      mid=(low+high)/2
      if ip10[mid] > ip :
         high = mid - 1 #Continue find in R[low..mid-1]
      else:
         if int(ip20[mid])<int(ip) :
            low = mid + 1 #Continue find in R[mid+1..high]
         else:
            return mid # Find success,return the index value.
return 0

def ip2country():
print 'start loading ...'
sum = 0
d = {}

try:
      fpath = sys.argv[1] #the first argv is log file
      opath = sys.argv[2] #the second argv is output file
except:
      print usage
      return 0

#loading ip data.
load_data()
print 'start match ip .'

if os.path.isfile(fpath):
      fh = open(fpath,'r')
      for line in fh:
         sum += 1
         lines=line.split(' ')
         ip = str(ip2int(lines[0]))
         c_id = get_country_id(ip)
         try:
            try:
                  d[country[c_id]] += 1
            except:
                  d[country[c_id]] = 1
         except:
            try:
                  d['other'] += 1
            except:
                  d['other'] =1
      fh.close()

sd=sortbyvalue(d)

fl = open(opath,'w')
for lt in sd:
      fl.write(lt[0] + ' , ' + str(lt[1]) + '/n')
fl.close
print 'all time -> ' + str(time()-stime)
print 'all line -> ' + str(sum)

def test():
sc = SqliteConnection("ip2country.db")
sc_sql = ""
sc_row = ''

ip = '3728668051'
sc_sql = "select * from ip2country where ip10<='"
sc_sql += ip + "' and ip20>'" + ip + "' order by ip20"
temp = sc.execute(sc_sql)

print temp[0][2].decode(encoding)

if name == "main":
ip2country()
…………………………………………………

Python 在目录中查找指定的字符串

在目录中查找指定的字符串

# -*- coding: cp936 -*-
'''在python中查找字符串，如果文件中包含指定的字符串，
则将此字符串所在的行号和文件名称打印出来,不支持正则表达式，
演示了walk使用方法；
演示了fileinput使用方法'''
import os
# 使用walk来遍历目录
def processFiles(arg,dirpath,files):
'''walk函数的回调函数，
查找满足条件的文件，使用fileinput对文件进行搜索，
如果查找到关键字，打印出行号和文件名称'''
for f in files:
      fname = os.path.join(dirpath,f)
      if(os.path.isfile(fname)):
         if os.path.splitext(fname)[1] == '.py':
            import fileinput
            import sys
            # 使用fileinput来遍历文件，如果有Tk则将此【行号：文件名称】插入搜索结果列表框
            for line in fileinput.input(fname):
                  # 在这里使用vTextFind，查找指定的字符串
                  if line.find(root.vTextFind.get()) != -1:
                     root.lbResults.insert(END,"[%d]:[%s]" % (fileinput.lineno(),fname))

def updateListbox(master):
'''搜索按钮的事件处理函数,
这个函数调用walk来遍历目录，并将搜索结果插入列表框'''
print master.vDirSearched.get()
master.lbResults.delete(0,END)
os.path.walk(master.vDirSearched.get(),processFiles,'')

from Tkinter import *
root = Tk()
# 创建搜索路径，并绑定变量，默认路径为当前'.'
Label(root,text = 'Directory to be searched').grid(row = 0,column = 0)
root.vDirSearched = StringVar()
root.vDirSearched.set('.')
root.etDirSearched = Entry(root,textvariable = root.vDirSearched)
root.etDirSearched.grid(row = 0,column = 1)

# 创建要搜索的字符串,默认搜索为'Tk'
Label(root,text = 'text to find').grid(row = 1,column = 0,stick = W)
root.vTextFind = StringVar()
root.vTextFind.set('Tk')
root.etTextFind = Entry(root,textvariable = root.vTextFind)
root.etTextFind.grid(row = 1,column = 1)

# 创建搜索按钮
root.btSearch = Button(root,
                     text = 'Search',
                     command = lambda
                     arg1 = root:
                     updateListbox(arg1)
                     )
root.btSearch.grid(row = 2,column = 0,columnspan = 2,stick = E + W)
# 创建搜索结果列表
root.lbResults = Listbox(root)
root.lbResults.grid(row = 3,column = 0,columnspan = 2,stick = E + W + N + S)

root.mainloop()