多页信息的爬取

爬取多个51job网页的信息

"""
爬取多页信息
1.找翻页规律，构造网址
2.通过循环爬取多页信息
"""
# 51job
"""
通过对比不同页的网址，找出规律
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,1.html
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,2.html
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,3.html
"""
import requests
import re
from pandas import DataFrame
import pandas as pd

jobInfoAll = DataFrame()

for i in range(1, 4):
    # 网址
    url = 'https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,'
          + str(i) + '.html'
    # 获得网页对象
    res = requests.get(url)
    # 设置编码格式
    res.encoding = "gbk"

    # 职位名
    # 将要提取的用(.*)表示，每条数据不同的地方用.*表示
    position_pat = '<a target="_blank" title="(.*)" href=".*" onmousedown=".*">'  
    position = re.findall(position_pat, res.text)
    # 公司名
    company_pat = '<span class="t2"><a target="_blank" title="(.*)" href=".*">.*</a></span>'
    company = re.findall(company_pat, res.text)
    # 工作地点
    place_pat = '<div class="el">.*?<span class="t3">(.*?)</span>'  # 非贪婪模式
    place = re.findall(place_pat, res.text, re.S)
    # 薪资
    salary_pat = '<div class="el">.*?<span class="t4">(.*?)</span>'  # 非贪婪模式
    salary = re.findall(salary_pat, res.text, re.S)

    # 将取出的信息放到数据框
    jobInfo = DataFrame([position, company, place, salary]).T
    # 设置列名
    jobInfo.columns = ['职位名', '公司名', '工作地点', '薪资']
    jobInfoAll = pd.concat([jobInfoAll, jobInfo])

# 数据的条数
print(len(jobInfoAll))

# 将数据保存到本地
jobInfoAll.to_csv('51job3.csv')