作用:可以在请求中重写headers
pip3 install fake_useragent
导入fake_useragent 包
from fake_useragent import UserAgent
class RandomUserAgentMiddlerware(object):
def __init__(self,crawler):
super(RandomUserAgentMiddlerware,self).__init__()
self.ua = UserAgent()
self.ua_type = crawler.settings.get('RANDOM_UA_TYPE','random')
@classmethod
def from_crawler(cls,crawler):
return cls(crawler)
def process_request(self,request,spider):
def get_ua():
return getattr(self.ua,self.ua_type)
request.headers.setdefault("User-Agent",get_ua())
scrapy在middlewares.py中添加随机user-agent
最新推荐文章于 2021-08-03 23:53:20 发布