import requests
from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
import re
# 将操作码保存为txt文件
def text_save(filename, data): # filename为写入txt文件的路径,data为要写入数据列表.
file = open(filename, 'w')
for i in range(len(data)):
s = str(data[i]).replace('[', '').replace(']', '') # 去除[],这两行按数据不同,可以选择
s = str(data[i]).replace('(', '').replace(')', '')
s = s.replace("'", '').replace(',', '') + '\n' # 去除单引号,逗号,每行末尾追加换行符
file.write(s)
file.close()
print("保存文件成功")
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
}#爬虫[Requests设置请求头Headers],伪造浏览器
# 核心爬取代码
url="http://www.shanghairanking.cn/rankings/bcur/2020"
ret=Request(url,headers=headers)
html=urlopen(ret)
bs=BeautifulSoup(html,"html.parser")
tr = bs.find('tbody').find_all('tr')
#.find定位到所需数据位置 .find_all查找所有的tr(表格)
listall=[]
# 去除标签栏
for j in tr[0:]: #tr[1:]遍历第1列到最后一列,表头为第0列
td = j.find_all('td')#td表格
date = td[0].get_text().strip() #遍历排名
college_name = td[1].get_text().strip() #遍历学校名称
region = td[2].get_text().strip() #遍历省市
type = td[3].get_text().strip() #遍历类型
score = td[4].get_text().strip() #遍历总分
a = j.find_all('a') #a表格
file = a[0].get('href') #遍历链接
link = "http://www.shanghairanking.cn{0}".format(file)
print(date, college_name, region, type, score, link)
list = date, college_name, region, type, score, link
listall.append(list)
file = []
list = []
text_save('University Rankings.txt', listall)
Python爬虫学习之爬取学校排名写入txt文件
最新推荐文章于 2022-01-26 16:09:27 发布