Python ——下载GPM IMERG数据

需要自行修改的部分：账号，密码，保存路径，文件名，IDM路径

import urllib
import os
import urllib.request
import numpy as np
import re
import requests
from subprocess import call
from multiprocessing import Pool
from subprocess import Popen
from getpass import getpass
import platform
import os
import shutil
import tempfile
from http import cookiejar
from urllib import request
from urllib.parse import urlencode

#创建文件.netrc  .dodsrc （非必需）
'''
urs = 'urs.earthdata.nasa.gov'    # Earthdata URL to call for authentication
prompts = ['Enter NASA Earthdata Login Username \n(or create an account at urs.earthdata.nasa.gov): ',
           'Enter NASA Earthdata Login Password: ']

homeDir = os.path.expanduser("~") + os.sep

with open(homeDir + '.netrc', 'w') as file:
    file.write('machine {} login {} password {}'.format(urs, getpass(prompt=prompts[0]), getpass(prompt=prompts[1])))
    file.close()
with open(homeDir + '.urs_cookies', 'w') as file:
    file.write('')
    file.close()
with open(homeDir + '.dodsrc', 'w') as file:
    file.write('HTTP.COOKIEJAR={}.urs_cookies\n'.format(homeDir))
    file.write('HTTP.NETRC={}.netrc'.format(homeDir))
    file.close()

print('Saved .netrc, .urs_cookies, and .dodsrc to:', homeDir)

# Set appropriate permissions for Linux/macOS
if platform.system() != "Windows":
    Popen('chmod og-rw ~/.netrc', shell=True)
else:
    # Copy dodsrc to working directory in Windows  
    shutil.copy2(homeDir + '.dodsrc', os.getcwd())
    print('Copied .dodsrc to:', os.getcwd())
'''

# IDM下载
def idmDownloader(task_url, folder_path, file_name):
    """
    IDM下载器
    :param task_url: 下载任务地址
    :param folder_path: 存放文件夹
    :param file_name: 文件名
    :return:
    """
    # IDM安装目录
    idm_engine = "D:/Internet Download Manager/IDMan.exe"
    # 将任务添加至队列
    call([idm_engine, '/d', task_url, '/p', folder_path, '/f', file_name, '/q'])
    # 开始任务队列
    call([idm_engine, '/s'])
    

def Downloader(URL,savepath,dataname):
    password_manager = request.HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password(None, "https://urs.earthdata.nasa.gov", 'NASA账号', 'NASA密码')#改为自己的帐号密码
    cookie_jar = cookiejar.CookieJar()
    opener = request.build_opener(request.HTTPBasicAuthHandler(password_manager),request.HTTPCookieProcessor(cookie_jar))
    request.install_opener(opener)
    myrequest = request.Request(URL)
    response = request.urlopen(myrequest)
    response.begin()
    #local_output = tempfile.NamedTemporaryFile(delete=False)
    FILENAME=os.path.join(savepath,dataname)
    try:
        f = open(FILENAME,'wb')
        f.write(response.read())
        f.close()
        print(dataname+"下载完成！")
    except:
        print(dataname+"下载失败！")
        idmDownloader(URL, savepath, dataname)#如果python下载失败，调用IDM下载
''' 
下载链接示例    
https://gpm1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi?FILENAME=%2Fdata%2FGPM_L3%2FGPM_3IMERGHH.06%2F2000%2F153%2F3B-HHR.MS.MRG.3IMERG.20000601-S000000-E002959.0000.V06B.HDF5&DATASET_VERSION=06&BBOX=-90%2C-180%2C90%2C180&VERSION=1.02&VARIABLES=precipitationCal&SHORTNAME=GPM_3IMERGHH&LABEL=3B-HHR.MS.MRG.3IMERG.20000601-S000000-E002959.0000.V06B.HDF5.SUB.nc4&FORMAT=bmM0Lw&SERVICE=L34RS_GPM
'''
def download(URL):
	#根据自己的链接自行更改，通过re.findall函数截取链接中的字符，具体用法请自行学习
    date=re.findall(r'MRG.3IMERG.*?-S', URL)[0][-10:-2]#获取文件的日期（年月日）
    savepath="E:/GPM IMERG/"+date[:4]+"/"+date[4:6]+"/"+date[6:]#文件的保存路径：路径+年+月+日，也就是把每一天的文件放一起
    if not os.path.exists(savepath):#创建保存文件的文件夹
        os.makedirs(savepath)
    dataname=re.findall(r'=3B.*?nc4', URL)[0][1:]#保存的文件名字
    if os.path.exists(os.path.join(savepath,dataname)):
        print(dataname + "exist")
    else:
        Downloader(URL,savepath,dataname)
        
if __name__ == '__main__':
    txtfile="E:/GPM IMERG/subset_GPM_3IMERGHH_06_20230527_023705_.txt"#获取的存有下载链接的txt文件
    with open(txtfile,'r') as f:
        urls=f.readlines()
    urls=np.array(urls)
    po=Pool(5)#一次读取几个链接
    po.map(download,urls)
    po.close()
    po.join()

下载结果：