selenium切换到div_selenium爬取拉勾网职位信息

前倾概要

拉勾网的反爬虫做可以说非常好。。。。。用一些常规的反反爬虫方法拿不到数据。然后这次就利用selenium模仿用户行为去获取数据。

源码

import re
from lxml import etree
from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import  csv
'''
python学习群：1004391443
'''
class  LagouSpider(object):
    driver_path = r'D:chromedriverchromedriver.exe'

    def __init__(self):
        self.driver = webdriver.Chrome(executable_path=self.driver_path)
        self.url = 'https://www.lagou.com/jobs/list_%E8%BF%90%E7%BB%B4?px=default&city=%E6%B7%B1%E5%9C%B3#filterBox'
        self.positions = []  # 用来存放所有职位信息

    def run(self):
        self.driver.get(self.url)
        count = 1
        while True:
            print('正在获取第{}页的数据'.format(count))
            count += 1
            self.parse_detail_page(self.driver.page_source)
            # 如果页数是最后一页，则退出循环
            if re.search(r'action="next" class="pager_next pager_next_disabled"',self.driver.page_source):
                break
            self.next_page() # 点击进入下一页
        self.driver.quit()  # 将浏览器退出
        self.write_to_csv()  # 将获取的数据写入文件



    def parse_detail_page(self,html):
        source_html = etree.HTML(html)  # 解析页面
        detail_list = source_html.xpath('//a[@class="position_link"]/@href')  # 找到每一页里面的职位详情页链接
        self.driver.execute_script("window.open()")  # 开启新的标签页
        self.driver.switch_to.window(self.driver.window_handles[1])  # 切换到新的标签页
        for url in detail_list:   # 遍历职位的详情页
            self.driver.get(url)  # 打开职位的详情页
            detail_url = etree.HTML(self.driver.page_source)  # 解析详情页
            name = re.findall(r'<span class="name">([^<]*)',self.driver.page_source)[0]  # 得到职位名称
            advantage = re.findall(r'职位诱惑：.*?<p>([^<]*)',self.driver.page_source,re.DOTALL)[0] # 得到职位诱惑内容
            job_request = detail_url.xpath('//dd[@class="job_request"]/p//span/text()')
            salary = job_request[0] # 获取薪资
            request = re.sub('[/]','',','.join(job_request[1:]))
            job_descript = detail_url.xpath('//div[@class="job-detail"]//p/text()')  # 获取职位描述
            job_descript = ' '.join(job_descript)
            address = re.findall(r'<input type="hidden" name="positionAddress" value="([^"]*)',self.driver.page_source)[0] # 获取工作地点
            position = {  # 将获取到的数据存入字典
                'name':name,
                'address':address,
                'advantage':advantage,
                'salary':salary,
                'request':request,
                'job_descript':job_descript
            }
            sleep(1)  # 睡一下 以防开启太快被临时封ip
            self.positions.append(position)  # 将存放数据的字典添加到列表中
            print(position)

        self.driver.close() # 关闭标签页
        self.driver.switch_to.window(self.driver.window_handles[0]) # 切换页面

    def next_page(self):
        # 找到下一页标签
        element = WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((By.CLASS_NAME, "pager_next")))
        element.click() # 点击下一页标签
        sleep(1)

    def write_to_csv(self):  # 写入文件
        header = ['name', 'address','advantage','salary','request','job_descript']
        with open('positons.csv','w',newline='',encoding='utf-8') as fp :
            write = csv.DictWriter(fp,header)
            write.writeheader()
            write.writerows(self.positions)




if __name__ == '__main__':
    spider = LagouSpider()
    spider.run()

运行结果:
只是一部分

文件：