由于手动查找检索某一关键字的论文很费时间,所以从网上查找了一些爬虫的相关代码,看是否可以一次性爬取想要的论文。但由于网上的代码都是把所有论文下载下来的,并没有匹配自己想要的论文。所以写了一个能匹配论文题目某一关键字的爬虫代码来爬取相关论文。
# -*- coding: utf-8 -*-
"""
Created on Thu Dec 6 09:35:58 2018
@author: M
"""
import re
import requests
import urllib.request
import os
import argparse
parser = argparse.ArgumentParser(description="test")
parser.add_argument('--keyword',type=str,default='detection') #传参匹配我们想要查找论文的关键字
args = parser.parse_args()
# get web context
r = requests.get('http://openaccess.thecvf.com/CVPR2018.py')
data = r.text
# find all pdf links
link_list = re.findall(r"(?<=href=\").+?pdf(?=\">pdf)|(?<=href=\').+?pdf(?=\">pdf)" ,data)
name_list = re.findall(r"(?<=href=\").+?2018_paper.html\">.+?</a>" ,data)
cnt = 1
num = len(link_list)
# your local path to download pdf files
localDir = './CVPR2018/{}/'.format(args.keyword)
if not os.path.exists(localDir):
os.makedirs(localDir)
while cnt < num:
url = link_list[cnt]
# seperate file name from url links
file_name = name_list[cnt].split('<')[0].split('>')[1]
# to avoid some illegal punctuation in file name
file_name = file_name.replace(':','_')
file_name = file_name.replace('\"','_')
file_name = file_name.replace('?','_')
file_name = file_name.replace('/','_')
file_name = file_name.replace(' ','_')
search_list = file_name.split('_')
search_pattern = re.compile(r'{}'.format(args.keyword),re.IGNORECASE)
download_next_paper = True
# print([True for i in search_list if search_pattern.findall(i)])
if ([True for i in search_list if search_pattern.findall(i)]):
download_next_paper = False
if download_next_paper:
cnt = cnt + 1
continue
file_path = localDir + file_name + '.pdf'
if os.path.exists(file_path):
print('File 【{}.pdf】 exists,skip downloading.'.format(file_name))
cnt = cnt + 1
continue
else:
# download pdf files
print('['+str(cnt)+'/'+str(num)+"] Downloading -> "+file_path)
try:
urllib.request.urlretrieve('http://openaccess.thecvf.com/'+url,file_path)
except :
cnt = cnt + 1
continue
cnt = cnt + 1
print("all download finished")
参考链接:https://blog.csdn.net/a529975125/article/details/79479438