【爬虫实战】python爬取中国最好大学排行榜

       最近闲来无事，帮朋友做了个python小作业，爬取武书连2020中国最好大学排名榜单。比较简单，我就直接上代码了。

代码展示

import requests  # 请求库
from bs4 import BeautifulSoup  # 解析库
import xlwt  # xls库
import random  # 随机数库

# 构造请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
    'Connection': 'close',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Referer': 'https://www.dxsbb.com/news/list_75.html',
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language':'zh-CN,zh;q=0.9'
}
# 代理列表
user_agent_list = [
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
    ]
# 随即更换代理
headers['User-Agent'] = random.choice(user_agent_list)

# 目标网站网址
url = "https://www.dxsbb.com/news/46702.html"
# 使用requests库爬取
res = requests.get(url=url,headers=headers)
# 设置文字格式
res.encoding = res.apparent_encoding
# 解析网页文件
soup = BeautifulSoup(res.text, 'html.parser')
# 获取所有大学排名
tbody = soup.find('tbody')
universities = tbody.find_all('tr')
# 用列表存储所有学校
universitiesInfos = []
# 遍历所有大学，获取信息
for university in universities:
    # 字典存储每个学校具体信息
    universitiesInfo = {'rank': '', 'rank_province': '', 'name': '', 'province': ''}
    # 全国排名
    rank = university.find_all('td')[0].text
    universitiesInfo['rank'] = rank
    # 分省排名
    rank_province = university.find_all('td')[1].text
    universitiesInfo['rank_province'] = rank_province
    # 学校名称
    name = university.find_all('td')[2].text
    universitiesInfo['name'] = name
    # 所在省份
    province = university.find_all('td')[3].text
    universitiesInfo['province'] = province
    # 保存
    universitiesInfos.append(universitiesInfo)

# 创建一个Excel文件
file = xlwt.Workbook()
# 创建sheet工作表
sheet1 = file.add_sheet(u'武书连2020中国最好大学排名', cell_overwrite_ok=True)

# 将数据来源网址url写入第一行
sheet1.write(0,0,"数据来源")
sheet1.write(0,1,url)
# 写入数据从第二行开始
for i in range(len(universitiesInfos)):
    # 每行写入全国排名、省份排名、学校名称、所在省份信息
    sheet1.write(i+1, 0, universitiesInfos[i]['rank'])  # 第i行，第1列
    sheet1.write(i+1, 1, universitiesInfos[i]['rank_province'])
    sheet1.write(i+1, 2, universitiesInfos[i]['name'])
    sheet1.write(i+1, 3, universitiesInfos[i]['province'])

# 设置第三列的宽度
sheet1.col(2).width = 5000
# 保存文件
file.save("武书连2020中国最好大学排名.xls")
print('\n***xsl文件下载完成***')

最终效果