boss爬虫（scrapy+selenium）

由于boss的反爬虫策略，我又没有使用ip代理，所以爬的比较慢

环境：pyCharm+jupyter notebook

from scrapy import signals
from selenium import webdriver
import time
import scrapy
from scrapy.http.response.html import HtmlResponse
from selenium.webdriver.common.action_chains import ActionChains
import requests

import pandas as pd

class CookiesMiddlewares(object):
    def __init__(self):
        print("初始化浏览器")
        self.driver = webdriver.Chrome()

    def process_request(self, request):
        self.driver.get(request.url)
        time.sleep(5)
        
        action = ActionChains(self.driver)
        nav_tag_list  = self.driver.find_elements_by_xpath(".//span[@class='job-name']")
        for tag in nav_tag_list:
            #print(tag.text)
            #模拟移动鼠标获得动态加载后的数据
            action.move_to_element(tag).perform()
            time.sleep(5)
        #获取页面的源码   
        source = self.driver.page_source