remotesensing批量下载

受人所托，要把remotesensing期刊所有的文章下载下来，看了下是开放获取，从09年到现在共12期，不算太多。思路为爬取下载链接，之后再IDM中下载，Url爬取程序为：

# https://www.mdpi.com/2072-4292/1/1/22/pdf # pdf下载链接
# https://www.mdpi.com/2072-4292/3/8  # 卷/期
# https://www.mdpi.com/2072-4292/vol(1-11)/Issue(1-12)/ # 主页面
import requests,traceback
from bs4 import BeautifulSoup as bs
base_url = 'https://www.mdpi.com'

def getDownUrl(url):
    headers = {'User-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
    results = []
    resp = requests.get(url,headers = headers)
    text = resp.text
    soup = bs(text,'html.parser')
    ais = soup.select('a[class="UD_Listings_ArticlePDF"]')
    for a in ais:
        r = base_url+a['href']
        results.append(r)
    return results
    
def writeUrl(f,results):
    for r in results:
        f.write(r+'\n')
    f.flush()
    
if __name__=='__main__':
	# 下载链接保存到downloadUrl.txt中
    f = open("downloadUrl.txt","w")
    results = []
    #url = 'https://www.mdpi.com/2072-4292/3/8'
    for i in range(1,13):
        for j in range(1,13):
            url = 'https://www.mdpi.com/2072-4292/{}/{}'.format(i,j)
            print(url)
            try:
                r = getDownUrl(url)
                results += r
                writeUrl(f,r)
            except:
                traceback.print_exc()   
    for r in results:
        print(r)