使用selenium抓取boss直聘

# -*- coding:utf-8 -*-
from selenium import webdriver
from  lxml import etree
import time

class BossSpider(object):
    def __init__(self):
        self.start_url = 'https://www.zhipin.com/job_detail/?query=python&city=101020100&industry=&position='
        self.driver = webdriver.Chrome()

    def get_page_url(self,source):
        html = etree.HTML(source)
        position_href_links = html.xpath("//div[@class='job-list']/ul/li//div[@class='info-primary']/h3/a/@href")
        for href in position_href_links:
            position_url = "https://www.zhipin.com" + href
            print(position_url)
            self.get_detail_info(position_url)
            time.sleep(2)
        time.sleep(2)

    def get_detail_info(self,url):
        self.driver.execute_script('window.open("%s")'%url)
        self.driver.switch_to.window(self.driver.window_handles[1])
        source = self.driver.page_source
        self.parse_detail_page(source)
        self.driver.close()
        self.driver.switch_to.window(self.driver.window_handles[0])

    def parse_detail_page(self,source):
        html = etree.HTML(source)
        position_info = {}
        position_info["company"] = html.xpath("//div[@class='company-info']/a[2]/text()")[0].strip()
        # position_info["salary"] = html.xpath("//div[@class='info-primary']/div[@class='name']/span/text()]")[0].strip()
        position_info["salary"] = html.xpath("//span[@class='salary']/text()")[0].strip()
        position_info["city"] = html.xpath("//div[@class='info-primary']/p//text()")[0]
        position_info["work_years"] = html.xpath("//div[@class='info-primary']/p//text()")[1]
        position_info["education"] = html.xpath("//div[@class='info-primary']/p//text()")[2]
        position_info["desc"] = "".join(html.xpath("//div[@class='job-sec']/div/text()")).strip()
        print(position_info)

    def run(self):
        self.driver.get(self.start_url)
        while True:
            source =self.driver.page_source
            self.get_page_url(source)
            next_btn = self.driver.find_element_by_xpath("//div[@class='page']/a[last()]")
            if "next disabled" in next_btn.get_attribute("class"):
                pass
            else:
                next_btn.click()
                time.sleep(2)
            time.sleep(5)

if __name__ == '__main__':
    BossSpider().run()