scrapy自定义重试

 

1.通过响应的 状态码或者异常来进行重试

class MyselfSpiderMiddleware(RetryMiddleware):

    def process_response(self, request, response, spider):
        if request.meta.get('dont_retry', False):
            return response
        if response.status in self.retry_http_codes:
            spider.get_ip()
            xs_root_dir = XS_ROOT_DIR
            if not os.path.exists(xs_root_dir):
                os.makedirs(xs_root_dir)
            # with open(os.path.join(xs_root_dir, 'error_url.txt'), 'a+', encoding='utf-8') as file:
            #     file.write(request.url + '\n')
            reason = response_status_message(response.status)
            return self._retry(request, reason, spider) or response
        return response

    def process_exception(self, request, exception, spider):
        if (
            isinstance(exception, self.EXCEPTIONS_TO_RETRY)
            and not request.meta.get('dont_retry', False)
        ):
            spider.get_ip()
            xs_root_dir = XS_ROOT_DIR
            if not os.path.exists(xs_root_dir):
                os.makedirs(xs_root_dir)
            # with open(os.path.join(xs_root_dir, 'error_url.txt'), 'a+', encoding='utf-8') as file:
            #     file.write(request.url + '\n')
            return self._retry(request, exception, spider)

 

 

2.自定义不满足爬虫需求的进行重试：

    def parse(self, response):
        assert isinstance(response,scrapy.http.HtmlResponse)
        section_list_ele = response.xpath('//div[@class="box_con"]/div[@id="list"]/dl/dd/a')
        xiaoshuo_name = response.meta['xiaoshuo_name']
        print(xiaoshuo_name)

        if len(section_list_ele) == 0:
            print(response.url)
            retries = response.meta.get('cus_retry_times', 0) + 1
            if self.RETRY_ENABLED:
                if retries <= self.RETRY_TIMES:
                    print('当前重试第%s次！'%retries)
                    r = response.request.copy()
                    r.meta['cus_retry_times'] = retries
                    r.dont_filter = True
                    yield r
        else:
            for index,i in enumerate(section_list_ele):
                assert isinstance(i, scrapy.selector.unified.Selector)
                title = i.xpath("./text()").get()
                href = i.attrib['href']
                item = FirstproItem()
                item['xiaoshuo_name'] = xiaoshuo_name
                item['title'] = str(index+1).zfill(4)+title
                item['href'] = href
                # yield item
                #yield response.follow(i,callback=self.parse_detail,meta={'item':item})   #与下面方法二选一
                yield scrapy.Request(url=response.urljoin(href),callback=self.parse_detail,meta={'item':item})

 

 

特别提醒：如果想使用自己的重试方式，则需要将默认的重试方式给禁用，查看方法：

'scrapy.downloadermiddlewares.retry.RetryMiddleware': None