动态页面,selenium和scrapy对接

 

#middlewares.py
class WebDriverMiddleware(object):

    def process_request(self, request, spider):
        print("++++++++++++++下载中间件被启用++++++++++++++++++")
        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        browser = webdriver.Chrome(chrome_options=options)
        browser.get(request.url)#加载网页，下载动态内容
        data = browser.page_source#拿到网页源码
        data = data.encode('utf-8')#对内容进行编码
        browser.quit()
        #返回给响应对象，返回到爬虫里的parse方法
        return HtmlResponse(request.url,body=data,encoding='utf-8',request=request)

   
#settings.py

DOWNLOADER_MIDDLEWARES = {
   'bilibili.middlewares.WebDriverMiddleware': 543,
}