爬取多个51job网页的信息
"""
爬取多页信息
1.找翻页规律,构造网址
2.通过循环爬取多页信息
"""
# 51job
"""
通过对比不同页的网址,找出规律
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,1.html
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,2.html
https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,3.html
"""
import requests
import re
from pandas import DataFrame
import pandas as pd
jobInfoAll = DataFrame()
for i in range(1, 4):
# 网址
url = 'https://search.51job.com/list/000000,000000,0000,32,9,99,Java%25E5%25BC%2580%25E5%258F%2591,2,'
+ str(i) + '.html'
# 获得网页对象
res = requests.get(url)
# 设置编码格式
res.encoding = "gbk"
# 职位名
# 将要提取的用(.*)表示,每条数据不同的地方用.*表示
position_pat = '<a target="_blank" title="(.*)" href=".*" onmousedown=".*">'
position = re.findall(position_pat, res.text)
# 公司名
company_pat = '<span class="t2"><a target="_blank" title="(.*)" href=".*">.*</a></span>'
company = re.findall(company_pat, res.text)
# 工作地点
place_pat = '<div class="el">.*?<span class="t3">(.*?)</span>' # 非贪婪模式
place = re.findall(place_pat, res.text, re.S)
# 薪资
salary_pat = '<div class="el">.*?<span class="t4">(.*?)</span>' # 非贪婪模式
salary = re.findall(salary_pat, res.text, re.S)
# 将取出的信息放到数据框
jobInfo = DataFrame([position, company, place, salary]).T
# 设置列名
jobInfo.columns = ['职位名', '公司名', '工作地点', '薪资']
jobInfoAll = pd.concat([jobInfoAll, jobInfo])
# 数据的条数
print(len(jobInfoAll))
# 将数据保存到本地
jobInfoAll.to_csv('51job3.csv')