优雅的操作scrapy爬虫的开始和结束
scrapy 管道中开始和结束信号
from scrapy import signals
class MySpider(object):
def __init__(self):
pass
@classmethod
def from_crawler(cls, crawler):
self = cls()
crawler.signals.connect(self.x1, signal=signals.spider_opened) # 绑定信号发生时允许的函数
crawler.signals.connect(self.x2, signal=signals.spider_closed)
return self
# 爬虫开始执行此函数
def x1(self, spider):
print('open')
# 爬虫结束,执行此函数
def x2(self, spider):
print('close')
###############################
# settings.py EXTENSIONS = {
'xdb.ext.MyExtend':666,
}
其中可选信号为:
# 信号可选类型 from scrapy import signals 中可以看到
engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()
scrapy spider中开始和结束信号
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
class MySpider(CrawlSpider):
def __init__(self):
# 将信号与函数绑定
dispatcher.connect(self.spider_closed, signals.spider_closed)
# 爬虫结束时执行的函数
def spider_closed(self, spider):
pass
# 2.*的scrapy写法改了
from scrapy import signals
from scrapy import Spider
class DmozSpider(Spider):
name = "dmoz"
allowed_domains = ["dmoz.org"]
start_urls = [
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def spider_closed(self, spider):
spider.logger.info('Spider closed: %s', spider.name)
def parse(self, response):
pass
spider中可选择的信号为:
# 可选信号
engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()