#!/usr/bin/python#-*- coding: utf-8 -*-#######################################################################Purpose: auto download kernel module patch from offical web#Useage: ./patch_spider.py#example: patch_spider.py linux-3.10.y fs/ubifs 2013-08-15:2016-11-11#Version: Initial Version by Alex######################################################################
importhttplib, urllib, urllib2importsys, osimportdatetimeimportre
DEBUG=0if DEBUG: from pprint import *
classPatchSpider(object):def __init__(self):pass
defrun(self):#main function
print ‘get patches info...‘patch_ids=self.get_patch_id()print ‘saving patches into...‘ret=self.save_patch_file(patch_ids)print ‘done‘
return0defget_patch_id(self):#get online data
response_data =[]
urlstr= "http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/log/%s?h=%s&ofs=%d"dt_format= ‘%Y-%m-%d‘start, end= args[‘date-range‘].split(‘:‘)if not start.strip(): start =datetime.datetime.now().strftime(dt_format)if not end.strip(): end =datetime.datetime.now().strftime(dt_format)
dt_start=datetime.datetime.strptime(start, dt_format)
dt_end=datetime.datetime.strptime(end, dt_format)for ofs in range(0, 10000, 50):
url= urlstr % (args[‘module‘], args[‘version‘], ofs)
req=urllib2.Request(url)try:
response=urllib2.urlopen(req).read()for item inself.extract_from(response):
date, id, msg=item
dt_date=datetime.datetime.strptime(date, dt_format)if dt_start <= dt_date <=dt_end:printdate, id, msg
response_data.append(item)else:returnresponse_dataexcepturllib2.HTTPError, e:print(‘HTTPError =‘ +str(e.code))excepturllib2.URLError, e:print(‘URLError =‘ +str(e.reason))excepthttplib.HTTPException, e:print(‘HTTPException‘)exceptException:raise
returnresponse_datadefsave_patch_file(self, l):
format= ‘http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/patch/%s?id=%s‘folder= self.assemble_path(args[‘module‘])ifos.path.exists(folder):print ‘%s existed, please backup your data!‘ %folderreturn -1
else:
os.makedirs(folder)
log= ["%s %s %s\n" %(i[0],i[1],i[2]) for i inl]
open(os.path.join(folder,‘patches.log‘), ‘w‘).writelines(log)
total=len(l)for index, item inenumerate(l):
_,id,msg=item
patch= format % (args[‘module‘], id)
file= os.path.join(folder, self.assemble_path(msg) + ‘.patch‘)
urllib.urlretrieve(patch, file)print "[%d/%d] save %s" %(index+1, total, file)defextract_from(self, html):
res= r"""
.*?(?P\d+-\d+-\d+).*?
\w+)‘>(?P.*?).*?
"""rec= re.compile(res, re.DOTALL|re.VERBOSE)returnre.findall(rec,html)defassemble_path(self, msg):path= ‘‘
for c inmsg:
path+= c if c.isalnum() else ‘_‘
returnpath#default args
args ={"version": "linux-3.10.y","module": "fs/ubifs","date-range": "2013-08-15:",
}if len(sys.argv) > 1:
args[‘version‘] = sys.argv[1]if len(sys.argv) > 2:
args[‘module‘] = sys.argv[2]if len(sys.argv) > 3:
args[‘date-range‘] = sys.argv[3]
ret=PatchSpider().run()
sys.exit(ret)