1.使用requests库模拟浏览器发送请求,接收响应功能
2
# 在dos命令行执行:
# pip show requests # 查看requests是否安装,以及相关信息
# # 没有安装的使用下面的语句进行安装
# pip install requests
总体分为三步
# 1 获取源数据 requests
# 2 解析源数据 bs4
# 3 保存数据 json/csv
#首先在文件中导入requests库
import requests
# 设置请求头变量
# 浏览器f12打开开发者界面,找到网络里面的请求头
headers={
'Host': 'www.zhipin.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0',
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
# 'Accept-Encoding': 'gzip, deflate, br',
# 'Referer': 'https://www.zhipin.com/job_detail/?ka=header-job',
# 'Connection': 'keep-alive',
# 'Upgrade-Insecure-Requests': 1,
# 'Cache-Control': 'max-age=0',
}
url='https://www.zhipin.com/job_detail/?query=python%E5%BC%80%E5%8F%91&scity=101010100&industry=&position='
# 模拟浏览器发送一个请求,返回一个响应
resp=requests.get(url,headers=headers)
# 使用响应对象的.text进行查看响应的文档形式
# 使用上下文管理协议with来以写的方式打开一个文件,给文件对象命名为f
with open('boss.txt',mode='w',encoding='utf-8') as f:
f.write(resp.text)
# 2 解析源数据 bs4
# pip install bs4
from bs4 import BeautifulSoup as bs
soup=bs(resp.text,'html.parser')
print(type(soup))
print(soup.title)
print(soup.title.name)
print(soup.title.string)
print(soup.title.text)
def data_get(url):
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:61.0) Gecko/20100101',
}
r=requests.get(url,headers=headers)
if r.status_code==200:
return r.text
# print(dir(r))
#解析源数据函数
def data_parse(data):
res=[]
soup=bs(data,'html.parser')
# 找到页面所有class=job-primary的div标签
all_job=soup.find_all(attrs={'class':'job-primary'})
for i in all_job:
job={}
job['job_name']=i.find(attrs={'class':'job-title'}).text
job_salary=i.find(attrs={'class','red'}).text.split('-')
job['job_min'],job['job_max']=job_salary
job['job_company']=i.find(attrs={'class':'company-text'}).h3.text
res.append(job)
return res
# 保存数据,先将数据转换为json格式的字符串,然后再保存到文件中
# import json
# def data_save(data):
# with open('boss.json',mode='w',encoding='utf-8') as f:
# f.write(json.dumps(data))
import csv
def data_save_csv(data):
with open('boss.cvs',mode='w',encoding='gbk',newline='') as f:
cw=csv.DictWriter(f,data[0].keys())
cw.writerows(data)
data=data_get('https://www.zhipin.com/job_detail/?query=python%E5%BC%80%E5%8F%91&scity=101010100&industry=&position=')
all_data=data_parse(data)
# for i in all_data:
# print(i)
# data_save(all_data)
data_save_csv(all_data)
未完待续.............