最近偶然搞到一份ChemDraw的学习资料,一共84个文件,挨个下载也不麻烦,但是本人比较好折腾,写了一个爬虫,搞到了全部资料。这个山寨网盘还是做了一些反爬虫工作的,所以这个代码花了好长时间。另外发现这个山寨网盘似乎并没有把所有资料都显示出来,所以通过这个爬虫还搞到了额外收获。废话少说,上代码:
"""
爬取ChemDraw资料
author: 王琪璇
date: 2020/1/14
"""
import requests
import json
import re
import time
def validateTitle(title):
"""
将不合法文件名进行替换处理,函数源自CSDN
:param title: 原文件名
:return: 替换后文件名
"""
rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/ \ : * ? " < > |'
new_title = re.sub(rstr, "_", title) # 替换为下划线
return new_title
cookiejar = requests.cookies.RequestsCookieJar()
headerdict = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/79.0.3945.117 Safari/537.36"} # 修改UA以防止网站反爬虫监测
req = requests.get("https://www.lanzous.com/b789488/", headers=headerdict, cookies=cookiejar)
cookiejar.update(req.cookies)
htmlcode = req.text
reg = re.search("var pgs;\\s*var [^ ]* = '([^']*)';\\s*var [^ ]* = '([^']*)';", htmlcode)
time.sleep(1.5) # 如果不在这里等待,会被网站反爬虫检测
for i_page in range(1, 3):
pg = requests.post("https://www.lanzous.com/filemoreajax.php",
data={"lx": 2, "fid": 789488, "pg": i_page, "t": reg.group(1), "k": reg.group(2), "pwd": "bxit"},
headers=headerdict, cookies=cookiejar)
cookiejar.update(pg.cookies)
js = json.loads(pg.text)
for i_item in js["text"]:
itm = requests.get("https://www.lanzous.com/%s" % i_item["id"], headers=headerdict, cookies=cookiejar)
htmlcode = itm.text
regs = re.findall('src="/fn\\?([^"]*)"', htmlcode)
refurl = "https://www.lanzous.com/fn?%s" % regs[1]
pgn = requests.get(refurl, headers=headerdict, cookies=cookiejar)
htmcode = pgn.text
regs = re.findall("var sg = '([^']*)';", htmcode)
if len(regs) == 0:
regs = re.findall("'sign':'([^']*)'", htmcode)
for j in regs:
if not j == "t":
sg = j
break
else:
sg = regs[0]
new_h = headerdict
new_h["Referer"] = refurl
pgns = requests.post("https://www.lanzous.com/ajaxm.php",
data={"action": "downprocess", "sign": sg, "ves": "1"}, headers=new_h, cookies=cookiejar)
del sg
jsu = json.loads(pgns.text)
print(i_item["name_all"])
file_url = "https://vip.d0.baidupan.com/file/%s" % jsu["url"] #自己去新建一个文件夹
new_h["Accept-Language"] = "zh-CN,zh;q=0.9"
file_get = requests.get(file_url, headers=new_h)
with open("file/" + validateTitle(i_item["name_all"]), "wb") as file_w:
file_w.write(file_get.content)