谷歌学术首页url爬取

小Demo

• 因为涉及到js加密，所以写了练练手
• 直接上代码

import requests
import re
import execjs

url = 'https://ac.scmor.com/'

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36', 
}

response = requests.get(url=url,headers=headers)
response.encoding = 'utf-8'
page_text = response.text
# print(page_text)

# 因为需要解析出的数据不是标签信息，所以用了正则
pattern = re.compile('autourl\=\[(.*?)\]',re.S)
tag = re.findall(pattern,page_text)[0]
detail_pattern = re.compile('\"(.*?)\"')
tag_list = re.findall(detail_pattern,tag)
print(tag_list)
node = execjs.get()

cxt = node.compile(open("./JsTest1.js",encoding='utf8').read())
for i in tag_list:
    js = 'strdecode("{0}")'.format(i)
    ret_url = cxt.eval(js)
    print(ret_url)

解析结果