优雅的操作scrapy爬虫的开始和结束

scrapy 管道中开始和结束信号

from scrapy import signals

class MySpider(object):
    def __init__(self):
        pass
    @classmethod
    def from_crawler(cls, crawler):
        self = cls()
        crawler.signals.connect(self.x1, signal=signals.spider_opened) # 绑定信号发生时允许的函数
        crawler.signals.connect(self.x2, signal=signals.spider_closed)
        return self
    # 爬虫开始执行此函数
    def x1(self, spider):
        print('open')
    # 爬虫结束，执行此函数
    def x2(self, spider):
        print('close')
###############################
# settings.py EXTENSIONS = {
            'xdb.ext.MyExtend':666,
        }

其中可选信号为：

# 信号可选类型  from scrapy import signals 中可以看到
engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()

scrapy spider中开始和结束信号

from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher

class MySpider(CrawlSpider):
    def __init__(self):
     	# 将信号与函数绑定
        dispatcher.connect(self.spider_closed, signals.spider_closed) 
    # 爬虫结束时执行的函数
    def spider_closed(self, spider): 
        pass


# 2.*的scrapy写法改了
from scrapy import signals
from scrapy import Spider


class DmozSpider(Spider):
    name = "dmoz"
    allowed_domains = ["dmoz.org"]
    start_urls = [
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
    ]


    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider


    def spider_closed(self, spider):
        spider.logger.info('Spider closed: %s', spider.name)


    def parse(self, response):
        pass

spider中可选择的信号为：

# 可选信号
engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()