项目四：数据获取：拉勾网

点击跳转到总目录

本篇只记录程序点击跳转项目

直接上代码

• 用selenium参考网上大神们做的

from selenium import webdriver
import time
import logging
import random
import openpyxl

wb = openpyxl.Workbook()   # 创建工作薄对象
sheet = wb.active          # 获取活动的工作表
# 添加列名
sheet.append(['job_name', 'company_name', 'city','industry', 'salary', 'experience_edu','welfare','job_label'])
# 输出日志的基本配置
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s')


def search_product(key_word):
    browser.find_element_by_id('cboxClose').click()     # 关闭让你选城市的窗口
    time.sleep(2)
    browser.find_element_by_id('search_input').send_keys(key_word)  # 定位搜索框 输入关键字
    browser.find_element_by_class_name('search_button').click()     # 点击搜索
    browser.maximize_window()    # 最大化窗口
    time.sleep(2)
    browser.find_element_by_class_name('body-btn').click()    # 关闭弹窗  啥领取红包窗口
    time.sleep(random.randint(1, 3))
    browser.execute_script("scroll(0,3000)")      # 下拉滚动条
    get_data()           # 调用抓取数据的函数
    # 模拟点击下一页   翻页爬取数据  每爬取一页数据  休眠   控制抓取速度  防止被反爬 让输验证码
    for i in range(10):
        browser.find_element_by_class_name('pager_next ').click()
        time.sleep(1)
        browser.execute_script("scroll(0,3000)")
        get_data()
        time.sleep(random.randint(3, 5))


def get_data():
    # Xpath提取想要的数据
    items = browser.find_elements_by_xpath('//*[@id="s_position_list"]/ul/li')
    for item in items:
        job_name = item.find_element_by_xpath('.//div[@class="p_top"]/a/h3').text
        company_name = item.find_element_by_xpath('.//div[@class="company_name"]').text
        city = item.find_element_by_xpath('.//div[@class="p_top"]/a/span[@class="add"]/em').text
        industry = item.find_element_by_xpath('.//div[@class="industry"]').text
        salary = item.find_element_by_xpath('.//span[@class="money"]').text
        experience_edu = item.find_element_by_xpath('.//div[@class="p_bot"]/div[@class="li_b_l"]').text
        welfare = item.find_element_by_xpath('.//div[@class="li_b_r"]').text
        job_label = item.find_element_by_xpath('.//div[@class="list_item_bot"]/div[@class="li_b_l"]').text
        data = f'{job_name},{company_name},{city},{industry},{salary},{experience_edu},{welfare},{job_label}'
        logging.info(data)
        sheet.append([job_name, company_name, city,industry, salary, experience_edu, welfare, job_label])


def main():
    # 访问目标url  https://www.lagou.com/
    browser.get('https://www.lagou.com/')
    time.sleep(random.randint(1, 3))
    search_product(keyword)
    # 保存数据
    wb.save('job_info.xlsx')


if __name__ == '__main__':
    # 待输入的关键词
    keyword = 'Python 数据分析'
    # chromedriver.exe的路径
    # chrome_driver =  webdriver.Chrome(executable_path="E:\Google\Chrome\Application\chromedriver.exe")
    options = webdriver.ChromeOptions()
    # 关闭左上方 Chrome 正受到自动测试软件的控制的提示
    options.add_experimental_option('useAutomationExtension', False)
    options.add_experimental_option("excludeSwitches", ['enable-automation'])
    browser = webdriver.Chrome(options=options, executable_path="E:\Google\Chrome\Application\chromedriver.exe")
    main()
    browser.quit()    # 关闭浏览器