基于python_爬取51前程无忧-贵阳数据分析岗位

爬虫—51前程无忧

import csv, re, time
from bs4 import BeautifulSoup
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import Chrome, ChromeOptions

f = open('./贵阳数据分析.csv', 'a', encoding='utf-8', newline='')
writer = csv.writer(f)
writer.writerow(['岗位', '发布时间','薪资', '公司', '地址','经验', '其他'])

b = Chrome()


def get_net_data():

    b.get('https://www.51job.com/')
    search_input = b.find_element_by_css_selector('#kwdselectid')
    input('是否继续')
    search_input.send_keys('数据分析')
    search_input.send_keys(Keys.ENTER)

    while True:
        # 获取页信息
        page_div = b.find_element_by_css_selector('.rt.rt_page')
        pages = page_div.text.split('/')
        all_page = int(pages[-1])
        current_page = int(pages[0])

        # 获取网页源代码
        print(b.page_source)
        save_data(b.page_source)

        if current_page < 12:
            next = b.find_element_by_css_selector('.next')
            next.click()
        else:
            break


def save_data(html: str):
    soup = BeautifulSoup(html, 'lxml')
    time.sleep(2)
    all_job_div = soup.select('.j_joblist>.e')
    one_page_jobs = []
    for job_div in all_job_div:
        name = job_div.select_one('.jname.at').get_text()
        sal = job_div.select_one('.sal').get_text()
        company = job_div.select_one('.cname.at').get_text()
        job_url = job_div.select_one('.el').attrs['href']
        experience =job_div.select_one('.d.at').get_text()
        experience = experience.replace('  |  ', '-')
        Release_time = job_div.select_one('.time').get_text()
        Release_time = Release_time.replace('发布', ' ')
        # print(experience)
        other = job_div.select_one('.tags')
        if other:
            other = other.attrs['title']
        else:
            other = None
            print(Release_time)
        one_page_jobs.append([name,Release_time, sal, company, job_url, experience, other])
    writer.writerows(one_page_jobs)


if __name__ == '__main__':
    get_net_data()