#他们网站的源代码变过多次,所以重写了爬取代码
完整代码:
import requests
import re
def getHtmlText(url):
try:
r=requests.get(url,timeout=30)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
print("getHtmlText err")
def parsePage(ulist,html):
try:
ranking=re.findall(r'\"ranking\"\:\"[\d]*\"',html)
univNameCn=re.findall(r'\"univNameCn\"\:\".*?\"',html)
scores=re.findall(r'\"score\"\:[\d]*',html)
for i in range(30):
rank=eval(ranking[i].split(":")[1])
name=eval(univNameCn[i].split(":")[1])
score=eval(scores[i].split(":")[1])
ulist.append([rank,name,score])
except:
print("parsePage err")
def printGoodsList(ulist):
tplt = "{0:^10}{1:{3}^10}{2:^10}"
print(tplt.format("序号", "学校名称", "分数", chr(12288))) # 中文空格填充,能保证输出对齐
for i in range(len(ulist)):
u = ulist[i]
print(tplt.format(u[0], u[1], u[2], chr(12288)))
def main():
url = "https://www.shanghairanking.cn/api/pub/v1/bcur?bcur_type=11&year=2020"
ulist = []
html = getHtmlText(url)
parsePage(ulist,html)
printGoodsList(ulist)
main()
使用https://www.shanghairanking.cn/_nuxt/static/1627561984/rankings/bcur/2021/payload.js的代码。
import requests
import re
def getHtmlText(url):
try:
r=requests.get(url,timeout=30)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
print("getHtmlText err")
def parsePage(ulist,html):
try:
#ranking=re.findall(r'\"ranking\"\:\"[\d]*\"',html)
univNameCn=re.findall(r'univNameCn\:\".*?\"',html)
scores=re.findall(r'score\:[\d]*',html)
for i in range(30):
#rank=eval(ranking[i].split(":")[1])
name=eval(univNameCn[i].split(":")[1])
score=eval(scores[i].split(":")[1])
ulist.append([name,score])
except:
print("parsePage err")
def printGoodsList(ulist):
tplt = "{0:^10}{1:{3}^10}{2:^10}"
print(tplt.format("序号", "学校名称", "分数", chr(12288))) # 中文空格填充,能保证输出对齐
for i in range(len(ulist)):
u = ulist[i]
print(tplt.format(i, u[0], u[1], chr(12288)))
def main():
url = "https://www.shanghairanking.cn/_nuxt/static/1627561984/rankings/bcur/2021/payload.js"
ulist = []
html = getHtmlText(url)
parsePage(ulist,html)
printGoodsList(ulist)
main()
1、新的url怎么获取
直接查看页面源代码,并没有发现我们想要获取的信息。于是,按f12打开开发者工具;在network栏目下,刷新一下页面,按ctrl+f打开搜索框,随便输入任意一个大学名称,就可以发现我们真正想要的内容很可能藏在/api/pub/v1/bcur?bcur_type=11&year=2020文件中。
点击左边搜索到的链接,再点右边的preview,就能发现这个是我们要的url
从headers里面复制url
2、对获取的内容进行分析
import requests
url = "https://www.shanghairanking.cn/api/pub/v1/bcur?bcur_type=11&year=2020"
html = requests.get(url)
html.encoding = html.apparent_encoding
print(html.text)
执行以上代码可以得到的信息如下:
3、选择技术路径
由于我们要的信息都在键值对中,所以适合的方案不是beautifulsoup库,而是用re进行正则匹配。后面的思路可以参考淘宝爬取商品的课程