单个的结果都可以解析出来了,那就把数据整合一下,然后打印出来呗。
看来还是用列表吧。
每个学校一共几项:
1.排名,2.学校中文名,3.学校英文名,4.学校详情网址,5.所属地区,6.类型,7.总分,8.办学层次(默认是这个选项)。
每个学校一个列表,所有的学校再组合成一个大列表。
随便改了一下:仅供小白参考吧。
import requests
from bs4 import BeautifulSoup
import re
import bs4
def getHtmlText(url):
try:
r = requests.get(url)
r.raise_for_status() # 网页打开失败,触发异常
r.encoding = r.apparent_encoding
return r.text
except:
return "网页打开失败"
def getInfoFromHtml(uLIst,html):
soup = BeautifulSoup(html,"html.parser")
# param = re.compile(r"\d{1,3}")
# param1 = re.compile(r"\d{1,3}.\d?")
n = 0 # 记录有多少个
for it in soup.find_all("div", class_="ranking", string=True):
list = [re.search(r"\d+",it.string).group()]
uLIst.append(list)
n = 0
for it in soup.find_all("a", class_="name-cn", string=True):
uLIst[n].append(it.string)
n += 1
# print(it.string)
# print(it["href"])
n = 0
for it in soup.find_all("a", class_="name-en", string=True):
uLIst[n].append(it.string)
uLIst[n].append(it["href"])
# print(it.string)
# # print(it["href"])
n += 1
n = 0
for it in soup.find_all("img",alt=True,class_="univ-logo"):
uLIst[n].append(it["src"])
# print(it["alt"])
# print(it["src"])
# print(it["onerror"])
n += 1
n = 0
for it in soup.find_all("td", class_="align-left"):
for tag in it.find_next_siblings("td"):
uLIst[n].append(re.search(r"(\w.?)+",tag.text).group())
# print(re.search(r"(\w.?)+",tag.text).group())
n += 1
# print(uLIst)
return n
def printList(List,num=20):
print("{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}".format("排名","学校名称","学校英文名称","详细网址","校标网址","所属地区","类型","综合总分","办学层次"))
for i in range(num):
itm = List[i]
print("{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}".format(itm[0],itm[1],itm[2],itm[3],itm[4],itm[5],itm[6],itm[7],itm[8]))
if __name__ == '__main__':
infoList = []
try:
fp = open("url_save",mode="r",encoding="utf-8") # 尝试去打开文件,打开失败就重新下载
html = fp.read() # 全部读出来
if len(html) < 1000: # if 不需要使用括号,读取的内容小于1000个字节,也是重新下载
raise() # 人为触发异常
except: # 文件打开失败的时候,就触发重新爬取一次
print("文件打开失败,下载网页")
url = "https://www.shanghairanking.cn/rankings/bcur/2021"
html = getHtmlText(url)
# print(len(html))
fp = open("url_save", mode="w", encoding="utf-8") # 把爬取的内容写到文件中,供下一次运行使用
fp.write(html)
fp.close()
if len(html) > 1000:
num = getInfoFromHtml(infoList, html)
printList(infoList,num)
else:
print("文件内容小于1000个字节,不做解析处理")
打印结果:
粗略一看,好像没啥问题
写入到excel表格中存起来吧。
又去抄了一段代码:
import requests
from bs4 import BeautifulSoup
import re
# import bs4
import xlsxwriter as xw
def getHtmlText(url):
try:
r = requests.get(url)
r.raise_for_status() # 网页打开失败,触发异常
r.encoding = r.apparent_encoding
return r.text
except:
return "网页打开失败"
def getInfoFromHtml(uLIst,html):
soup = BeautifulSoup(html,"html.parser")
# param = re.compile(r"\d{1,3}")
# param1 = re.compile(r"\d{1,3}.\d?")
n = 0 # 记录有多少个
for it in soup.find_all("div", class_="ranking", string=True):
list = [re.search(r"\d+",it.string).group()]
uLIst.append(list)
n = 0
for it in soup.find_all("a", class_="name-cn", string=True):
uLIst[n].append(it.string)
n += 1
# print(it.string)
# print(it["href"])
n = 0
for it in soup.find_all("a", class_="name-en", string=True):
uLIst[n].append(it.string)
uLIst[n].append(it["href"])
# print(it.string)
# # print(it["href"])
n += 1
n = 0
for it in soup.find_all("img",alt=True,class_="univ-logo"):
uLIst[n].append(it["src"])
# print(it["alt"])
# print(it["src"])
# print(it["onerror"])
n += 1
n = 0
for it in soup.find_all("td", class_="align-left"):
for tag in it.find_next_siblings("td"):
uLIst[n].append(re.search(r"(\w.?)+",tag.text).group())
# print(re.search(r"(\w.?)+",tag.text).group())
n += 1
# print(uLIst)
return n
def printList(List,num=20):
print("{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}".format("排名","学校名称","学校英文名称","详细网址","校标网址","所属地区","类型","综合总分","办学层次"))
for i in range(num):
itm = List[i]
print("{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}{:^10}\t{:^10}\t{:^10}".format(itm[0],itm[1],itm[2],itm[3],itm[4],itm[5],itm[6],itm[7],itm[8]))
def saveListToExcel(LIst, fileName):
workbook = xw.Workbook(fileName) # 创建工作簿
worksheet1 = workbook.add_worksheet("sheet1") # 创建子表
worksheet1.activate() # 激活表
title = ["排名","学校名称","学校英文名称","详细网址","校标网址","所属地区","类型","综合总分","办学层次"] # 设置表头
worksheet1.write_row('A1', title) # 从A1单元格开始写入表头
k = 2
for it in LIst:
row = "A" + str(k)
worksheet1.write_row(row, it) # 从A2单元格开始写入数据
k += 1
workbook.close() # 关闭表
if __name__ == '__main__':
infoList = []
try:
fp = open("url_save",mode="r",encoding="utf-8") # 尝试去打开文件,打开失败就重新下载
html = fp.read() # 全部读出来
if len(html) < 1000: # if 不需要使用括号,读取的内容小于1000个字节,也是重新下载
raise() # 人为触发异常
except: # 文件打开失败的时候,就触发重新爬取一次
print("文件打开失败,下载网页")
url = "https://www.shanghairanking.cn/rankings/bcur/2021"
html = getHtmlText(url)
# print(len(html))
fp = open("url_save", mode="w", encoding="utf-8") # 把爬取的内容写到文件中,供下一次运行使用
fp.write(html)
fp.close()
if len(html) > 1000:
num = getInfoFromHtml(infoList, html)
printList(infoList,num)
saveListToExcel(infoList, "最好大学.xlsx")
else:
print("文件内容小于1000个字节,不做解析处理")
结果如下: