python 爬阳光高考高校数据

import json
import time
from requests_html import HTMLSession


# 爬取阳光高考院校库
def get_school_data():
    # 开启会话
    session = HTMLSession()
    # 初始化院校列表
    school_list = []
    # 分页参数，起始位置
    start = 1
    # 分页参数，每页条数，阳光高考院校库默认为20条/页
    limit = 10
    # 开始执行时的时间
    start_time = time.time()
    print("阳光高考院校库爬取任务")
    print("开始执行...")
    # 分页循环获取
    while True:
        # 阳光高考院校库查询网址
        # url = 'https://gaokao.chsi.com.cn/sch/search.do?searchType=1&start=' + str(start)
        url = 'https://gaokao.xdf.cn/college/china/searchSchool/_____'+str(start)
        # 获取请求到的html
        r = session.get(url)
        # 从中获取到院校库的table中的所有文字
        result = r.html.find("body > div.main > div.container > div.main-table > table > tbody > tr")

        # 初始化本次循环的数据列表
        this_list = []
        for res in result:
            text = res.text
        # 通过.将字符串拆分为数组，对本次循环获取到的分页数据进行循环处理
            # 每一条数据通过换行符拆分为数组
            arr = text.split("\n")
            # 初始化院校信息
            school = {}
            # 如果长度为1，则为通过.拆分时的最后一条无用信息，直接跳过
            if len(arr) == 1:
                continue

            if len(arr) == 9:
                school.update({"name": arr[0], "area": arr[1], "property": arr[2], "type": arr[3], "level": arr[4], "attrs": arr[5]})
            else:
                school.update({"name": arr[0], "area": arr[1], "property": "", "type": "", "level": "", "attrs": ""})
            # property = ""
            # if not arr[2]:
            #   property = arr[2]
            #
            # type = ""
            # if not arr[3]:
            #     type = arr[3]
            #
            # level = ""
            # if not arr[4]:
            #     level = arr[4]
            #
            # attrs = ""
            # if not arr[5]:
            #     attrs = arr[5]
            #
            # if property is None:
            #      property = ""
            #
            # if type is None:
            #      type = ""
            #
            # if level is None:
            #     level = ""
            #
            # if attrs is None:
            #     attrs = ""



            # 将院校追加到本次循环结果的数组中
            this_list.append(school)
            # 将院校追加到总的结果中
            school_list.append(school)
        print("爬取第 " + str(int(start)) + " 页，本页获取到 " + str(len(this_list)) + " 条数据")
        # 如果本次分页循环的长度小于分页的每页长度，说明本次为最后一页数据，数据读取完毕，终止循环
        if len(this_list) < limit:
            break
        # 否则起始位置需要加上一页
        start += 1
    # 执行完毕时的时间
    end_time = time.time()
    print("执行完毕，耗时 " + str(round(end_time - start_time, 2)) + " 秒，共获取到 " + str(len(school_list)) + " 条数据")
    # 关闭会话
    session.close()
    # 返回结果
    return school_list


# 保存为JSON数组
def save_json_data(school_list):
    data = open('./data.json', 'w')
    data.write(json.dumps(school_list, ensure_ascii=False, ).encode("utf8").decode())
    print("已保存为JSON数组(./data.json)")


# 保存为SQL可执行文件
def save_sql_file(school_list):
    school_sql = "insert into ant_user_school(name,area,property,type,level,attrs) values "
    for school in school_list:
        school_sql += "('" + school["name"] + "','" + school["area"] + "','" + school["property"] + "','" + school["type"] + "','" + school["level"] + "','" + school["attrs"] + "'),"
    if school_sql.endswith(","):
        school_sql = school_sql[0:-1]
    data = open('./data.sql', 'w')
    data.write(school_sql)
    print("已保存为SQL可执行文件(./data.sql)")


data = get_school_data()
# save_json_data(data)
save_sql_file(data)