Python爬虫实战之阳光高考专业库爬取

代码截止到2021.6.18

阳光高考的专业库界面的分析换了，这个代码目前来说用不了了，之后可能会更新

---

直接贴代码了，这不是库的教程，我直接贴代码了

吐槽一下，这网页的源码写的让我调的真难受

import requests
import re
import time


def get_html(url):
    headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
        'Connection': 'close'
    }

    data_ml = {  # 可用可不用
        'key': '1050'
    }

    r = requests.get(url=url, headers=headers)
    r.encoding = r.apparent_encoding
    time.sleep(0.8)  # 慢一点
    return r.text


def main():
    url_ml = "https://gaokao.chsi.com.cn/zyk/zybk/mlCategory.action?key=1050"  # 一级学科目录

    h5 = get_html(url_ml)
    xk = []  # 三级字典格式

    # print(h5)

    rule_num = re.compile('\d\d\d\d\d\d')
    ml_id = re.findall(pattern=rule_num, string=h5)

    rule_china = re.compile('[\u4E00-\u9FA5]{2,8}')
    ml_name = re.findall(string=h5, pattern=rule_china)

    for i in range(len(ml_id)):
        xk.append(
            {
                ml_name[i]: {}
            }
        )

    # print(xk)

    for id_ml in range(len(ml_id)):

        url_fenlei = "https://gaokao.chsi.com.cn/zyk/zybk/specialityesByCategory.action?key=" + ml_id[id_ml]
        h5 = get_html(url_fenlei)

        # 获取这个大类的所有专业
        rule = re.compile('\"_blank\">(.*)</a>')  # 正则匹配前后标签获取专业名称
        _ = re.findall(string=h5, pattern=rule)
        zhuan_ye = []
        for i in range(len(_)):
            if _[i] != '查看':
                zhuan_ye.append(_[i])
        zhuan_ye.pop(0)  # 去除大类名称

        # 这个大类的所有专业代码，包括大类码
        rule = re.compile('<td>(\d{6,})</td>')  # 匹配6位专业码
        daima = re.findall(string=h5, pattern=rule)

        # print(str(len(zhuan_ye)) + " " + str(len(daima)))
        # print(zhuan_ye)
        # print(daima)

        rule2 = re.compile('[\u4E00-\u9FA5]{,20}类$') #  正则表达式是在是调了老半天
        j = 0
        lei = ""
        f = 0
        for i in range(len(zhuan_ye)):
            if re.match(pattern=rule2, string=zhuan_ye[i]):
                xk[id_ml][ml_name[id_ml]][zhuan_ye[i]] = []
                lei = zhuan_ye[i]
                if f == 0:
                    j += 1
                    f = 1
            else:
                xk[id_ml][ml_name[id_ml]][lei].append(
                    (zhuan_ye[i], daima[j])
                )
                j += 1

    return xk
    # print(xk)


if __name__ == '__main__':
    main()