本文包含了几个本人常用的杂志的论文信息自动化抓取脚本。
一、【AIP】
AIP_Publishing
# 尝试通过网址获取页面文献信息并存入Excel,以爱思唯尔为例(基于selenium)
import re
import httpx
import requests
import pandas as pd
import openpyxl
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
def open_web(url):
requests.get(url)
headers = {
# cookie:用户信息,登录或不登录都有
'cookie': 'your cookie',
# 防盗链
'Referer': 'your Referer',
# user-agent:浏览器信息,版本,电脑'
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
}
response = requests.get(url, headers=headers, timeout=10, verify=False)
# print(response.text)
return response, headers
# 替换掉不能命名的字符
def replace_x(words):
words = words.replace('/', '')
words = words.replace(':', '')
words = words.replace('*', '')
words = words.replace('?', '')
words = words.replace('〈', '')
words = words.replace('〉', '')
words = words.replace('|', '')
return words
# 存储到Excel列表
def save_list(file_path_name, publication_time, Journal_name, url, title):
# 打开一个现有的工作簿
workbook = openpyxl.load_workbook(file_path_name)
# 选择一个工作表
sheet = workbook.active
# 创建一个字典
paper_info = {'time': publication_time, 'Journal': Journal_name, 'url': url, 'title': title}
# 将数据添加到下一个空行中
sheet.append(list(paper_info.values()))
# 保存工作簿
workbook.save(file_path_name)
# print(paper_info.values())
# 下载PDF
def Down_PDF(pdf_url, title, time, file_path, headers):
print(pdf_url)
PDF_title = '(' + time + ')' + title + '.pdf'
PDF_content = requests.get(url=pdf_url, headers=headers).content
with open(file_path + PDF_title, mode='wb') as f:
f.write(PDF_content)
# 主函数
file_path = 'D:\\paper\\'
while 1:
print('请输入文献网址(停止请按0)')
url = input()
if url == '0':
break
(response, headers) = open_web(url)
# 文章标题
title = re.findall('<title>(.*?)</title>', response.text)[0]
title = replace_x(title)
title = title.replace(' Physics of Fluids AIP Publishing', '')
print('标题:', title)
# 发布时间
publication_time = re.findall('<meta name="citation_publication_date" content="(.*?)" />', response.text)[0]
publication_time=publication_time[0:4]
print('时间:', publication_time)
# 杂志名称
Journal_name = 'Physics of Fluids'
print('杂志:', Journal_name)
print('(' + publication_time + ')' + title)
# 将所有信息添加到字典
print('是否保存?(1/0)')
is_save = input()
if int(is_save) == 1:
file_path_n