import requests
from bs4 import BeautifulSoup
from get_proxy import GetProxy
from urllib import parse
from day03.pymysql_text import Mysql_text
#请求头信息,放到全局变量方便使用
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
#定义一个函数用来获取boss直聘详情页url
def get_all_info(my_ip):
#第一个循环用来获取分页信息
for i in range(1,4):
url = 'https://www.zhipin.com/c101010100/h_101010100/?query=python&page=%s&ka=page-%s' %(i,i)
#这个循环是当你ip被封了之后,用except下面的函数获取新的代理ip
for i in range(4):
#如果ip失效就换新的代理ip
try:
#用BeautifulSoup获取我们想要的数据
response = requests.get(url,headers=headers,proxies=my_ip.proxy,timeout=5)
soup = BeautifulSoup(response.text,'lxml')
a_list = soup.select('div.info-primary > h3 > a ')
for a_ele in a_list:
#循环拼接详情页的路径
a_href = a_ele['href']
# print(a_href)
info_url = parse.urljoin(url,a_href)
print(info_url)
#这个循环个下面的try except是在ip失效之后获取新的ip
for i in range(4):
try:
#调用获取最终数据的函数
get_boss_info(my_ip, info_url)
#获取成功就break结束程序,避免代理ip的浪费
break
except Exception as e:
#获取新的代理ip
my_ip.updata_proxy()
print(e)
except Exception as e:
print(e)
my_ip.updata_proxy()
#这个函数用来获取我们想要的数据
def get_boss_info(my_ip,info_url):
response = requests.get(info_url,proxies = my_ip.proxy,headers=headers,timeout=5)
html_ele = BeautifulSoup(response.text,'lxml')
title = html_ele.select('h1')[0].text
# print(title)
price = html_ele.find('span',class_ = "badge").text.replace('\n','').strip()
# print(price)
address = html_ele.select('div.info-primary p')[0].text.replace('\n','').strip()
# print(address)
yaoqiu = html_ele.select('div.text')[0].text
# print(yaoqiu)
data = (title,price,address,yaoqiu)
print(data)
#实例化之前封装好的mysql类并实例化和执行数据添加
m = Mysql_text()
sql = 'insert into boss(title,price,address,yaoqiu) VALUES (%s,%s,%s,%s)'
m.sqlzz(sql,data)
if __name__ == '__main__':
#实例化获取代理的类的对象
my_ip = GetProxy()
#调用第一个函数
get_all_info(my_ip)
用代理的方式爬取boss直聘的信息
最新推荐文章于 2024-08-15 12:11:47 发布