middlewares部分(切记setting里打开该中间件)
前置步骤都放入spider_opened(在from_crawler里连接好)
import random
import logging
from scrapy import signals
from selenium.webdriver import Chrome
from scrapy.http import HtmlResponse
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
logger = logging.getLogger(__name__)
class SeleniumDownloadMiddleware:
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
def spider_opened(self):
chrome_options = Options()
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
self.driver = Chrome(chrome_options=chrome_options)
self.driver.maximize_window()
def spider_closed(self):
self.driver.quit()
def process_request(self, request, spider):
if request.meta.get('item') and request.meta.get('item').get('type'):
if request.meta.get('item').get('new_page'):
self.driver.get(request.url)
try:
js = "document.getElementById('bitAd_floatImage_max').style.display='none';"
self.driver.execute_script(js)
except:
pass
loc = ('css selector',
f'div.search-result-list>div[data-id=\'{request.meta["item"]["data_id"]}\']>p.cx-ck-count.text-hover>i')
try:
ele2 = WebDriverWait(self.driver, 6, 0.1).until(EC.presence_of_element_located(loc))
except:
# 防止第一次执行selenium时未进入指定网页
self.driver.get(request.url)
ele2 = WebDriverWait(self.driver, 6, 0.1).until(EC.presence_of_element_located(loc))
self.driver.execute_script("arguments[0].click();", ele2)
type_loc = ('css selector', 'span.ck-cx-list-wrapper>div.ck-cx-list-content>a>div>div')
WebDriverWait(self.driver, 10, 0.1).until(EC.presence_of_element_located(type_loc)).text # 更新页面元素的操作
html = self.driver.page_source
self.driver.execute_script("arguments[0].click();", ele2) # 复原页面
return HtmlResponse(request.url, body=html, encoding='utf8', request=request)
spider部分
进入下一个回调函数前,会进入刚才的中间件,执行process_request,selenium操作结束return response给spider,如果spider内有多个回调函数需要执行时,需要在selenium中间件里判断,否则每进入回调函数之前都会进入selenium中间件
class CarSpider(scrapy.Spider):
name = 'car'
allowed_domains = ['xxxxxxx']
start_urls = ['xxxxxxx']
def parse(self, response):
item = YichewangItem()
alpha_list = response.css('div.brand-list>div')
sort=1
for alpha in alpha_list:
item['sort']=sort
letter = alpha.css('div.item-letter::text').extract_first()
item['letter'] = letter
brand_list = alpha.css('div.item-brand')
sort+=1
for brand in brand_list:
brand_name = brand.css('a>div::text').extract_first()
logo = brand.css('img::attr(data-original)').extract_first()
item['brand_name'] = brand_name
item['logo'] = 'https://' + logo
id = brand.css('a>div::attr(data-id)').extract_first()
item['type'] = False
url = 'https://car.yiche.com/xuanchegongju/?mid=' + str(id)
yield scrapy.Request(url, callback=self.pages, meta={'item': copy.deepcopy(item)},dont_filter=True)