爬取某个山寨网盘上面的学习资料

最近偶然搞到一份ChemDraw的学习资料，一共84个文件，挨个下载也不麻烦，但是本人比较好折腾，写了一个爬虫，搞到了全部资料。这个山寨网盘还是做了一些反爬虫工作的，所以这个代码花了好长时间。另外发现这个山寨网盘似乎并没有把所有资料都显示出来，所以通过这个爬虫还搞到了额外收获。废话少说，上代码：

"""
    爬取ChemDraw资料
    author: 王琪璇
    date: 2020/1/14
"""
import requests
import json
import re
import time


def validateTitle(title):
    """
    将不合法文件名进行替换处理，函数源自CSDN
    :param title: 原文件名
    :return: 替换后文件名
    """
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(rstr, "_", title)  # 替换为下划线
    return new_title


cookiejar = requests.cookies.RequestsCookieJar()
headerdict = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                            "Chrome/79.0.3945.117 Safari/537.36"}  # 修改UA以防止网站反爬虫监测

req = requests.get("https://www.lanzous.com/b789488/", headers=headerdict, cookies=cookiejar)
cookiejar.update(req.cookies)
htmlcode = req.text
reg = re.search("var pgs;\\s*var [^ ]* = '([^']*)';\\s*var [^ ]* = '([^']*)';", htmlcode)
time.sleep(1.5)  # 如果不在这里等待，会被网站反爬虫检测

for i_page in range(1, 3):
    pg = requests.post("https://www.lanzous.com/filemoreajax.php",
                       data={"lx": 2, "fid": 789488, "pg": i_page, "t": reg.group(1), "k": reg.group(2), "pwd": "bxit"},
                       headers=headerdict, cookies=cookiejar)
    cookiejar.update(pg.cookies)
    js = json.loads(pg.text)
    for i_item in js["text"]:
        itm = requests.get("https://www.lanzous.com/%s" % i_item["id"], headers=headerdict, cookies=cookiejar)
        htmlcode = itm.text
        regs = re.findall('src="/fn\\?([^"]*)"', htmlcode)
        refurl = "https://www.lanzous.com/fn?%s" % regs[1]
        pgn = requests.get(refurl, headers=headerdict, cookies=cookiejar)
        htmcode = pgn.text

        regs = re.findall("var sg = '([^']*)';", htmcode)
        if len(regs) == 0:
            regs = re.findall("'sign':'([^']*)'", htmcode)
            for j in regs:
                if not j == "t":
                    sg = j
                    break
        else:
            sg = regs[0]
        new_h = headerdict
        new_h["Referer"] = refurl
        pgns = requests.post("https://www.lanzous.com/ajaxm.php",
                             data={"action": "downprocess", "sign": sg, "ves": "1"}, headers=new_h, cookies=cookiejar)
        del sg
        jsu = json.loads(pgns.text)
        print(i_item["name_all"])
        file_url = "https://vip.d0.baidupan.com/file/%s" % jsu["url"] #自己去新建一个文件夹
        new_h["Accept-Language"] = "zh-CN,zh;q=0.9"
        file_get = requests.get(file_url, headers=new_h)
        with open("file/" + validateTitle(i_item["name_all"]), "wb") as file_w:
            file_w.write(file_get.content)