pip install -U fake-useragent
- 如果以上方法不成功,则需要fake_useragent_0.1.11.json文件来处理了
- 参考[820]fake_useragent.errors.FakeUserAgentError: Maximum amount of retries reached_周小董-CSDN博客
- 下载链接UA.rarfake_useragent解决报错文件-Proxy文档类资源-CSDN下载
- 将fake_useragent_0.1.11.json文件复制到项目中,使用UserAgent类进行读取
-
from fake_useragent import UserAgent
# 配置headers
class RandomUserAgentMidddlware(object):
# 随机更换user-agent
def __init__(self, crawler):
super(RandomUserAgentMidddlware, self).__init__()
self.ua = UserAgent(path="spiders/fake_useragent_0.1.11.json")
# 从配置文件读取随机类型
self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
# 通过配置文件的随机类型进行调用
def get_ua():
return getattr(self.ua, self.ua_type)
request.headers.setdefault('User-Agent', get_ua())