最近在做scrapy项目的时候, 需要异步渲染动态的网页,自己构建了一个selenium 异步渲染的下载中间件
1.爬虫发送请求的时候需要携带参数
Request(
url=url,
meta={"is_selenium": True},
callback=self.parse,
dont_filter=True,
)
2.中间件代码
class SeleniumMiddleware(object):
"""selenium模拟渲染"""
def __init__(self):
self.options = webdriver.ChromeOptions()
self.options.add_argument('-headless')
self.options.add_experimental_option('excludeSwitches', ['enable-automation']) # 忽略证书错误
self.options.add_argument('--ignore-certificate-errors')
self.driver = webdriver.Chrome(executable_path="C:\chromedriver.exe", options=self.options)
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
def spider_closed(self):
self.driver.close()
def process_request(self, request, spider):
is_selenium = request.meta.get('is_selenium', False)
if is_selenium:
try:
self.driver.get(request.url)
except TimeoutException:
self.driver.execute_script('window.stop()')
return HtmlResponse(url=request.url, status=666, encoding='utf-8')
time.sleep(1)
url = self.driver.current_url
body = self.driver.page_source
return HtmlResponse(url=url, body=body, status=200, encoding='utf-8')
注意事项
使用中间件的时候注意资源的释放,需要监控爬虫结束的信号,在爬虫结束的时候,关闭掉selenium的driver。