项目代码:
import requests
import bs4
from bs4 import BeautifulSoup
def getHTMLText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ''
def fillUnivList(ulist, html):
soup = BeautifulSoup(html, 'html.parser')
for tr in soup.find('tbody').children:
if isinstance(tr, bs4.element.Tag):
tds = tr('td')
ulist.append([tds[0].string, tds[1].string, tds[4].string])
def printUnivList(ulist, num):
tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
print(tplt.format('排名', '学校名称', '总分', chr(12288)))
for i in range(num):
u = ulist[i]
print(tplt.format(u[0], u[1], u[2], chr(12288)))
def main():
uinfo = []
url = 'https://www.shanghairanking.cn/rankings/bcur/2020'
html = getHTMLText(url)
fillUnivList(uinfo, html)
printUnivList(uinfo, 35) # 20 univs
main()
问题描述:
错误信息如下:Traceback (most recent call last):
File "E:/Code/Python/爬虫/Python网络爬虫与信息提取/第二周/test.py", line 35, in <module>
main()
File "E:/Code/Python/爬虫/Python网络爬虫与信息提取/第二周/test.py", line 33, in main
printUnivList(uinfo, 35) # 20 univs
File "E:/Code/Python/爬虫/Python网络爬虫与信息提取/第二周/test.py", line 26, in printUnivList
print(tplt.format(u[0], u[1], u[2], chr(12288)))
TypeError: unsupported format string passed to NoneType.__format__
错误
原因分析:
用 .string 属性来提取标签里的内容时,该标签应该是只有单个节点的。string方法不能处理标签中含有标签的内容,否则返回为None。
而从网页源代码中可以看出,所要提取的信息都在嵌套在标签之间的
解决方案:
使用.text或者.get_text()方法获取信息即可,代码修改如下:def fillUnivList(ulist, html):
soup = BeautifulSoup(html, 'html.parser')
for tr in soup.find('tbody').children:
if isinstance(tr, bs4.element.Tag):
tds = tr('td')
ulist.append([tds[0].text, tds[1].text, tds[4].text])
可以看到如下的输出
但是格式并和预期并不一致,这是为什么呢?
从源码中得知,我们提取的信息包含很多多余的空格字符,使用strip()函数去掉即可
def fillUnivList(ulist, html):
soup = BeautifulSoup(html, 'html.parser')
for tr in soup.find('tbody').children:
if isinstance(tr, bs4.element.Tag):
tds = tr('td')
ulist.append([tds[0].text.strip(), tds[1].text.strip(), tds[4].text.strip()])
参考: