python的spider_python简易spider

#-*- encoding:utf-8 -*-importsgmllibimporturllib2importthreadingimportreimportsysfromtimeimport*SPIDER_ONINIT=0

SPIDER_ONDATA=1SPIDER_ONPARSER=2SPIDER_ONFINISH=3SPIDER_ONSTOP=4#this class is not used nowclassSpException(Exception):def__init__(self,msg):

self.message=msgdefMessage(self):printself.messageclassLog:def__init__(self,filename):

self.log=open(filename,'w')defWriteLog(self,msg):

self.log.write(msg)

self.log.flush()#golbalg_Log=Log(".\\spider_file\\spider.log")#this class is not usedclassResourcePool:def__init__(self):

self.res_list=[]

self.res_count=0

self.condition=threading.Condition()defAddResource(self,data):

self.condition.acquire()

self.res_list.append(data)

self.res_count=self.res_count+1self.condition.notify()

self.condition.release()defGetResource(self):