“揭秘CentosChina爬虫项目：掌握Scrapy框架的必备技巧与数据库设计“

最新推荐文章于 2024-09-12 23:06:08 发布

雾隐隐o

最新推荐文章于 2024-09-12 23:06:08 发布

阅读量652

点赞数 26

分类专栏：爬虫基础文章标签：爬虫 scrapy 数据库

本文链接：https://blog.csdn.net/m0_74087660/article/details/141031618

版权

爬虫基础专栏收录该内容

21 篇文章 0 订阅

订阅专栏

Centoschina

项目要求

爬取centoschina_cn的所有问题，包括文章标题和内容

数据库表设计

库表设计：

数据展示：

项目亮点

低耦合，高内聚。

爬虫专有settings

custom_settings = custom_settings_for_centoschina_cn

custom_settings_for_centoschina_cn = {
    'MYSQL_USER': 'root',
    'MYSQL_PWD': '123456',
    'MYSQL_DB': 'questions',
}

DownloaderMiddleware使用

class CentoschinaDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    # 处理请求
    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request 继续执行下一步操作，不处理默认返回None
        # - or return a Response object 直接返回响应, 如scrapy和pyppeteer不需要用下载器中间件访问外网，直接返回响应, pyppeteer有插件，一般和scrapy还能配合，selenium不行，没有插件
        # - or return a Request object 将请求返回到schdular的调度队列中供以后重新访问
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    # 处理响应
    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object 返回响应结果
        # - return a Request object 结果不对（判断结果对不对一般判断状态码和内容大小）一般返回request，也是将请求返回到schdular的调度队列中供以后重新访问
        # - or raise IgnoreRequest
        return response

    # 处理异常：如超时错误等
    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception 继续执行下一步，没有异常
        # - return a Response object: stops process_exception() chain 如果其返回一个 Response 对象，则已安装的中间件链的 process_response() 方法被调用。Scrapy将不会调用任何其他中间件的 process_exception() 方法。
        # - return a Request object: stops process_exception() chain 将请求返回到schdular的调度队列中供以后重新访问
        pass

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)

DownloaderMiddleware中抛弃请求写法

适用场景：请求异常，换代理或者换cookie等操作

# from scrapy.exceptions import IgnoreRequest
# raise IgnoreRequest(f'Failed to retrieve {request.url} after {max_retries} retries')

例子：处理下载异常并重试请求

import logging
from scrapy.exceptions import IgnoreRequest

class RetryExceptionMiddleware:
    def __init__(self):
        self.logger = logging.getLogger(__name__)

    def process_exception(self, request, exception, spider):
        # 记录异常信息
        self.logger.warning(f'Exception {exception} occurred while processing {request.url}')
        
        # 检查是否达到重试次数限制
        max_retries = 3
        retries = request.meta.get('retry_times', 0) + 1
        
        if retries <= max_retries:
            self.logger.info(f'Retrying {request.url} (retry {retries}/{max_retries})')
            # 增加重试次数
            request.meta['retry_times'] = retries
            return request
        else:
            self.logger.error(f'Failed to retrieve {request.url} after {max_retries} retries')
            raise IgnoreRequest(f'Failed to retrieve {request.url} after {max_retries} retries')

例子：切换代理

import random

class SwitchProxyMiddleware:
    def __init__(self, proxy_list):
        self.proxy_list = proxy_list
        self.logger = logging.getLogger(__name__)

    @classmethod
    def from_crawler(cls, crawler):
        proxy_list = crawler.settings.get('PROXY_LIST')
        return cls(proxy_list)

    def process_exception(self, request, exception, spider):
        self.logger.warning(f'Exception {exception} occurred while processing {request.url}')
        
        # 切换代理
        proxy = random.choice(self.proxy_list)
        self.logger.info(f'Switching proxy to {proxy}')
        request.meta['proxy'] = proxy
        
        # 重试请求
        return request

piplines中抛弃item写法

适用场景：数据清洗、去重、验证等操作

# from scrapy.exceptions import DropItem
# raise DropItem("Duplicate item found: %s" % item)

保存到文件(通过命令)

from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'centoschina_cn', '-o', 'questions.csv'])