scrapy 动态IP、随机UA、验证码

最新推荐文章于 2022-07-11 07:36:00 发布

weixin_33862514

最新推荐文章于 2022-07-11 07:36:00 发布

阅读量170

点赞数

文章标签： python

随机UA

https://github.com/hellysmile/fake-useragent

DOWNLOADER_MIDDLEWARES增加自定义

from fake_useragent import UserAgent

class RandomUserAgentMiddlware(object):
    # 随机更换user-agent
    def __init__(self, crawler):
        super(RandomUserAgentMiddlware, self).__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request, spider):
        def get_ua():
            return getattr(self.ua, self.ua_type)

        request.headers.setdefault('User-Agent', get_ua())

备注：settings.py增加配置项

RANDOM_UA_TYPE = "random"