爬取网址:www.51job.com
1.首先我们来分析需要爬取网站的情况
在51job中输入:数据分析师
具体过程详解,请移步:https://blog.csdn.net/qq_31848185/article/details/88967352
贴上我的过程,两种方式两种思路,没有好坏之分,共同学习,在此再一次感谢,前一位博主!
from requests_html import HTMLSession
import time
session = HTMLSession()
#定义一个url函数,自主输入要爬取的页数
def my_url(m):
while m>0:
p=m-(m-1)
url = 'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,{}.html'.format(p)
r = session.get(url)
for n in range(4,54):
workname = r.html.xpath(('//*[@id="resultList"]/div[{}]/p/span/a/@title').format(n))##职位
company=r.html.xpath(('//*[@id="resultList"]/div[{}]/span[1]/a/@title').format(n))#公司名
address=r.html.xpath(('//*[@id="resultList"]/div[{}]/span[2]/text()').format(n))#工作地点
money=r.html.xpath(('//*[@id="resultList"]/div[{}]/span[3]/text()').format(n))#薪资
time1 = r.html.xpath(('//*[@id="resultList"]/div[{}]/span[4]/text()').format(n)) # 薪资
print("第(",(n-3),")条:",workname,company,address,money)
time.sleep(1) # 时间间隔。防止被封IP
p=p+1
m=m-1
print(m)
print(p)
else:
return
if __name__=='__main__':
num=int(input('请输入要爬取的页数:'))#输入要爬取的页数
my_url(num)