Python爬虫实战之爬取拉勾网职位①

Github：https://github.com/why19970628/Python_Crawler/tree/master/LaGou

1.爬取拉钩招聘动态网页

网址：https://www.lagou.com/zhaopin/
通过分析网页结构，爬取当前网页的目标信息，以及各个网页内的职位描述

代码如下

import requests
from lxml import etree
import pandas as pd
from time import sleep
import random

# cookie
cookie = '你的cookie'
# headers
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
    'cookie':cookie
}
# 查看网页结构循环页数进行采集
for i in range(1, 20):
    sleep(random.randint(3, 10))
    url = 'https://www.lagou.com/zhaopin/jiqixuexi/{}/?filterOption=3'.format(i)
    print('正在抓取第{}页...'.format(i), url)
    # 请求网页并解析
    con = etree.HTML(requests.get(url=url, headers=headers).text)
    # 使用xpath表达式抽取各目标字段
    job_name = [i for i in con.xpath("//a[@class='position_link']/h3/text()")]
    job_address = [i for i in con.xpath("//a[@class='position_link']/span/em/text()")]
    job_company = [i for i in con.xpath("//div[@class='company_name']/a/text()")]
    job_salary = [i for i in con.xpath("//span[@class='money']/text()")]
    job_exp_edu = [i for i in con.xpath("//div[@class='li_b_l']/text()")]
    job_exp_edu2 = [i for i in [i.strip() for i in job_exp_edu] if i != '']
    job_industry = [i.strip() for i in con.xpath("//div[@class='industry']/text()")]
    job_tempation = [i for i in con.xpath("//div[@class='list_item_bot']/div[@class='li_b_r']/text()")]
    job_links = [i for i in con.xpath("//div[@class='p_top']/a/@href")]
    #print(job_links)

    # 获取详情页链接后采集详情页岗位描述信息
    job_des =[]
    for link in job_links:
        sleep(random.randint(3, 10))
        print('link:',link)
        con2 = etree.HTML(requests.get(url=link, headers=headers).text)
        #print(con)
        des = [[i for i in con2.xpath("//dd[@class='job_bt']/div/p/text()")]]
        job_des += des
    #print(job_des)
    break #遍历一次

# 对数据进行字典封装
dataset = {
    '岗位名称': job_name,
    '工作地址': job_address,
    '公司': job_company,
    '薪资': job_salary,
    '经验学历': job_exp_edu2,
    '所属行业': job_industry,
    '岗位福利': job_tempation,
    '任职要求': job_des
}

# 转化为数据框并存为csv
data = pd.DataFrame(dataset)
data.to_csv('machine_learning_hz_job2.csv')

数据展示

2.爬取静态网页

在请求Request URL网址时，返回”您操作太频繁，请稍后访问“，
用requests.Session()建立Session,建立完成session之后通过session来获取cookie

for page in range(1, pages):
        data = {
            'first': 'false',
            'pn': page,
            'kd': 'python'
        }
        s = requests.Session() # 建立session
        s.get(url=url1, headers=headers, timeout=3)
        cookie = s.cookies # 获取cookie
        respon = s.post(url = url, headers=headers, data=data, cookies=cookie, timeout=3)
        time.sleep(7)
        print(respon.text)

数据清洗网址

https://blog.csdn.net/weixin_43746433/article/details/94489298

数据分析与成图网址

https://blog.csdn.net/weixin_43746433/article/details/94500669