数据解析beautifulsoup爬取中国大学排名数据并保存到数据库（学习内容）

 

import requests

from bs4 import BeautifulSoup, builder

import pymysql

def download(url):

    headers={

        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40"

        }

    response=requests.get(url,headers=headers)

    return response

# 解析网页

def bs(url):

    html_text=download(url)

    html_text.encoding=html_text.apparent_encoding

    soup=BeautifulSoup(html_text.text,'lxml')

    #获取分数和省市

    sapList=soup.select('#content-box .rk-table tbody>tr')

    allsaps=[]

    for saptag in sapList:

        sap_dict={}

        sap_dict['province']=saptag.select('td')[2].get_text().strip()

        sap_dict['score']=saptag.select('td')[4].get_text().strip()

        sap_dict['rank']=saptag.select('div')[0].get_text().strip()

        sap_dict['name']=saptag.select('a')[0].get_text()

        sap_dict['type']=saptag.select('td')[3].get_text().strip()

        allsaps.append(sap_dict)

    print(allsaps)

    return allsaps

def get_config(host,user,password,db):

    db_config={

        'host':host,

        'user':user,

        'password':password,

        'db':db

    }

    return db_config

def getConn(allsaps):

    db_config=get_config('localhost','root','admin','python')

    conn=pymysql.Connect(**db_config)

    cur=conn.cursor()

    for allsap in allsaps:

        province=allsap['province']

        score=allsap['score']

        rank=allsap['rank']

        name=allsap['name']

        type=allsap['type']

        sql="INSERT INTO rank(num,schname,province,type,score)VALUES("+"'"+rank+"'"+","+"'"+name+"'"+","+"'"+province+"'"+","+"'"+type+"'"+","+"'"+score+"'"+")"

        cur.execute(sql)

    conn.commit()

    cur.close()

    conn.close()

    print("操作完成")

if __name__ == "__main__":

    url="https://www.shanghairanking.cn/rankings/bcur/2020"

    all=bs(url)

    getConn(all)