#!/usr/bin/env python#-*- coding:utf-8 -*-
from twisted.web.client importgetPage, deferfrom twisted.internet importreactorimportqueueclassResponse(object):def __init__(self, body, request):
self.body=body
self.request=request
self.url=request.url
@propertydeftext(self):return self.body.decode(‘utf-8‘)classRequest(object):def __init__(self, url, callback=None):
self.url=url
self.callback=callbackclassScheduler(object):def __init__(self, engine):
self.q=queue.Queue()
self.engine=enginedefenqueue_request(self, request):
self.q.put(request)defnext_request(self):try:
req= self.q.get(block=False)exceptException as e:
req=Nonereturnreqdefsize(self):returnself.q.qsize()classExecutionEngine(object):def __init__(self):
self._closewait=None
self.running=True
self.start_requests=None
self.scheduler=Scheduler(self)
self.inprogress=set()defcheck_empty(self, response):if notself.running:
self._closewait.callback(‘......‘)def_next_request(self):whileself.start_requests:try:
request=next(self.start_requests)exceptStopIteration:
self.start_requests=Noneelse:
self.scheduler.enqueue_request(request)while len(self.inprogress) < 5 and self.scheduler.size() > 0: #最大并发数为5
request=self.scheduler.next_request()if notrequest:breakself.inprogress.add(request)
d= getPage(bytes(request.url, encoding=‘utf-8‘))
d.addBoth(self._handle_downloader_output, request)
d.addBoth(lambdax, req: self.inprogress.remove(req), request)
d.addBoth(lambdax: self._next_request())if len(self.inprogress) == 0 and self.scheduler.size() ==0:
self._closewait.callback(None)def_handle_downloader_output(self, body, request):"""获取内容,执行回调函数,并且把回调函数中的返回值获取,并添加到队列中
:param response:
:param request:
:return:"""
importtypes
response=Response(body, request)
func= request.callback orself.spider.parse
gen=func(response)ifisinstance(gen, types.GeneratorType):for req ingen:
self.scheduler.enqueue_request(req)
@defer.inlineCallbacksdefstart(self):
self._closewait=defer.Deferred()yieldself._closewaitdefopen_spider(self, spider, start_requests):
self.start_requests=start_requests
self.spider=spider
reactor.callLater(0, self._next_request)classCrawler(object):def __init__(self, spidercls):
self.spidercls=spidercls
self.spider=None
self.engine=None
@defer.inlineCallbacksdefcrawl(self):
self.engine=ExecutionEngine()
self.spider=self.spidercls()
start_requests=iter(self.spider.start_requests())
start_requests=iter(start_requests)
self.engine.open_spider(self.spider, start_requests)yieldself.engine.start()classCrawlerProcess(object):def __init__(self):
self._active=set()
self.crawlers=set()def crawl(self, spidercls, *args, **kwargs):
crawler=Crawler(spidercls)
self.crawlers.add(crawler)
d= crawler.crawl(*args, **kwargs)
self._active.add(d)returnddefstart(self):
dl=defer.DeferredList(self._active)
dl.addBoth(self._stop_reactor)
reactor.run()def _stop_reactor(self, _=None):
reactor.stop()classSpider(object):defstart_requests(self):for url inself.start_urls:yieldRequest(url)classChoutiSpider(Spider):
name= "chouti"start_urls=[‘http://dig.chouti.com/‘,
]defparse(self, response):print(response.text)classCnblogsSpider(Spider):
name= "cnblogs"start_urls=[‘http://www.cnblogs.com/‘,
]defparse(self, response):print(response.text)if __name__ == ‘__main__‘:
spider_cls_list=[ChoutiSpider, CnblogsSpider]
crawler_process=CrawlerProcess()for spider_cls inspider_cls_list:
crawler_process.crawl(spider_cls)
crawler_process.start()