爬取拉勾网招聘信息(selenium+lxml)

#encoding: utf-8
from selenium import webdriver
from lxml import etree
import re
import time



class LagouSpider(object):
    driver_path = r"D:\Chromedriver\chromedriver.exe"
#初始化爬虫的时候，将需要的参数传入
    def __init__(self):
        self.driver = webdriver.Chrome(executable_path=LagouSpider.driver_path)
        self.url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
        self.positions = []

    def run(self):
        self.driver.get(self.url)
        while True:
            source = self.driver.page_source
            self.request_list_page(source)
            #爬取完一页的信息后，找到下一页的按钮
            nextbtn = self.driver.find_element_by_class_name("pager_next")
            #判断是否到达最后一页
            if "pager_next_disabled" in nextbtn.get_attribute("class"):
                pass
            else:
                nextbtn.click()
                time.sleep(1)
                
#根据搜索首页解析出对应职位的详细url
    def request_list_page(self,source):
        html = etree.HTML(source)
        links = html.xpath("//a[@class='position_link']/@href")
        for link in links:
            self.parse_detail(link)
            time.sleep(2)

#根据每个职位的详细url进行数据爬取分析
    def parse_detail(self,url):
        #另外打开详细职位的页面
        self.driver.execute_script("window.open('%s')"%url)
        #将driver定位到详细职位页面
        self.driver.switch_to.window(self.driver.window_handles[1])

        #使用xpath去解析数据
        source = self.driver.page_source
        htmlelement = etree.HTML(source)
        name = htmlelement.xpath('//h1[@class = "name"]/text()')[0]
        job_request = htmlelement.xpath('//dd[@class="job_request"]//span')
        salary = job_request[0].xpath('.//text()')[0].strip()
        city = job_request[1].xpath('./text()')[0].strip()
        city = re.sub(r"[\s/]", "", city)
        work_years = job_request[2].xpath('.//text()')[0].strip()
        work_years = re.sub(r"[\s/]", "", work_years)
        education = job_request[3].xpath('.//text()')[0].strip()
        education = re.sub(r"[\s/]", "", education)
        desciption = "".join(htmlelement.xpath("//dd[@class='job_bt']//text()"))
        dict = {
            'name': name,
            'salary': salary,
            'city': city,
            'work_years': work_years,
            'education': education,
            'desciption': desciption
        }
        self.positions.append(dict)
        print(dict)
        print('+'*30)


        #爬取完当前页面的信息后关闭当前页面
        self.driver.close()
        #将driver定位回搜索界面
        self.driver.switch_to.window(self.driver.window_handles[0])



if __name__ == '__main__':
    spider = LagouSpider()
    spider.run()

如果存在疑问，欢迎留言交流。