【2019.05】python 爬取拉钩数据（静态+动态）

拉钩这个网站有点意思，页面加载有两种方法
点击首页的标签，页面是静态加载的
直接搜索，页是动态加载的
两种爬取方式不一样，这里都实现了。

• 动态爬取时会出现 ‘操作太频繁，请稍后再试’字样，出现这种情况是cookie的问题，解决方案详见代码

静态爬取

https://www.lagou.com/zhaopin/jiqixuexi/?labelWords=label

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/5/6 21:12
# @Author  : Paulson
# @File    : Spider_jingtai.py
# @Software: PyCharm
# @define  : function


import random
import time

import pandas as pd
import requests
from lxml import etree

# 真实cookie
# Cookie = 'xx‘’
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    # 'Cookie': Cookie
}


for i in range(1, 2):
    time.sleep(random.randint(3, 10))
    url = 'https://www.lagou.com/zhaopin/jiqixuexi/{}/?filterOption=2'.format(i)
    print(url)
    print('正在抓取第{}页数据...'.format(i))

    # 请求网页并解析
    con = etree.HTML(requests.get(url, headers=headers).text)

    # 使用xpath表达式抽取各个字段信息
    job_name = [i for i in con.xpath('//a[@class="position_link"]/h3/text()')]
    print(job_name)
    job_address = [i for i in con.xpath("//a[@class='position_link']/span/em/text()")]
    job_company = [i for i in con.xpath("//div[@class='company_name']/a/text()")]
    job_salary = [i for i in con.xpath("//span[@class='money']/text()")]
    job_exp_edu = [i for i in con.xpath("//div[@class='li_b_l']/text()")]
    job_exp_edu2 = [i for i in [i.strip() for i in job_exp_edu] if i != '']
    job_industry = [i for i in con.xpath("//div[@class='industry']/text()")]
    job_tempation = [i for i in con.xpath("//div[@class='list_item_bot']/div[@class='li_b_r']/text()")]
    job_links = [i for i in con.xpath("//div[@class='p_top']/a/@href")]
    print(job_links)
    # 获取详情页链接后采集详情页岗位描述信息
    job_des = []
    for link in job_links:
        time.sleep(random.randint(3, 10))
        con2 = etree.HTML(requests.get(url=link, headers=headers).text)
        des = [[i.xpath('string(.)') for i in con2.xpath("//dd[@class='job_bt']/div/p")]]
        print(des)
        job_des += des

# 对数据进行字典封装
    dataset = {
        '岗位名称': job_name,
        '工作地址': job_address,
        '公司': job_company,
        '薪资': job_salary,
        '经验学历': job_exp_edu2,
        '所属行业': job_industry,
        '岗位福利': job_tempation,
        '任职要求': job_des
    }

# 转化为数据框并存为csv
    data = pd.DataFrame(dataset)
    data.to_csv('machine_learning_hz_job2.csv')

结果

动态爬取

https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/5/6 22:11
# @Author  : Paulson
# @File    : Spider_dongtai.py
# @Software: PyCharm
# @define  : function


import json
import random
import time
import requests
from bs4 import BeautifulSoup
import pandas as pd

#定义抓取主函数
def lagou_dynamic_crawl():
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Referer': 'https://www.lagou.com/jobs/list_%E8%BF%90%E7%BB%B4?city=%E6%88%90%E9%83%BD&cl=false&fromSearch=true&labelWords=&suginput=',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
    }

    # 创建一个职位列表容器
    positions = []
    for page in range(1, 31):
        print('正在抓取{}页数据...'.format(page))
        URL_ = 'https://www.lagou.com/jobs/list_%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput='
        URL = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
        # 构建请求保单参数
        params = {
            'first': 'false',
            'pn': page,
            'kd': '机器学习'
        }

        # 构造请求并返回结果
        s = requests.Session()
        s.get(URL_, headers=headers, timeout=3)  # 请求首页获取cookies
        cookie = s.cookies  # 为此次获取的cookies
        result = s.post(URL, headers=headers, data=params, cookies=cookie, timeout=3)
        print(result.text)

        # 将请求结果转化为json
        json_result = result.json()

        # 解析json数据获取目标信息
        try:
            position_info = json_result['content']['positionResult']['result']
        except:
            position_info = json_result['msg']
            if '您操作太频繁' in position_info:
                print('操作太频繁')
                assert False

        # 循环当前页每一个职位信息，再去爬职位详情页面
        for position in position_info:
            # 把我们要爬取的信息放入字典
            position_dict = {
                'position_name': position['positionName'],
                'work_year': position['workYear'],
                'education': position['education'],
                'salary': position['salary'],
                'city': position['city'],
                'company_name': position['companyFullName'],
                'address': position['businessZones'],
                'label': position['companyLabelList'],
                'stage': position['financeStage'],
                'size': position['companySize'],
                'advantage': position['positionAdvantage'],
                'industry': position['industryField'],
                'industryLables': position['industryLables']
            }
            # 找到职位id
            position_id = position['positionId']

            # 根据职位id调用岗位描述函数获取职位描述

            position_dict['position_detail'] = recruit_detail(position_id)
            positions.append(position_dict)

        time.sleep(random.randint(3, 6))
    print('全部数据采集完毕...')
    return positions

# 定义抓取岗位描述函数
def recruit_detail(position_id):
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
        "Cache-Control": "max-age=0",
        "Connection": "keep-alive",
        "Host": "www.lagou.com",
        "Referer": "https://www.lagou.com/jobs/list_%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0?labelWords=&fromSearch=true&suginput=",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
    }
    url = 'https://www.lagou.com/jobs/%s.html' % position_id
    result = requests.get(url, headers=headers)
    time.sleep(random.randint(1, 4))

    # 解析职位要求text
    soup = BeautifulSoup(result.text, 'html.parser')
    job_bt = soup.find(class_='job_bt')

    # 通过尝试发现部分记录描述存在空的情况
    # 所以这里需要判断处理一下
    if job_bt != None:
        job_bt = job_bt.text
    else:
        job_bt = 'null'
    return job_bt


if __name__ == '__main__':
    positions = lagou_dynamic_crawl()
    data = pd.DataFrame(positions)
    data.to_csv('machine_learning_hz_job3.csv')

结果

• 感谢star
  点击前往GitHub项目
  点击前往 jupyter notebook 文件