Python根据文本文档中的链接自动下载文件

背景

从Science Data Bank上找到了一个很好的数据，但是没有提供下载工具，只提供了每一个文件的网址，因此用python写了一个脚本来进行下载。
目前实现的功能：
1. 读取文本文档并下载
2. 自动添加延时，防止短时间进行过多请求导致服务器返回无效数据
3. 判断文件是否已下载，已下载则跳过，若文件大小过小则重新下载（因为无效数据会是体积很小的数据）

代码解析

1. 加载库

这里加载了所需的库

import requests   	# 用来发出get请求
import os		  	# 用来构建路径名
from urllib.parse import unquote  # 特殊字符需要转码才可以进行网页请求
import tqdm			# 进度条
import time			# 计算时间
import random		# 随机数，非固定时间间隔

2. 读取网址列表

直接读取，这里面的txt文件里面都是网址，内容举例：

https://download.scidb.cn/download?fileId=62295e16d7561b594fb67f58&dataSetType=personal&username=36680@qq.com&fileName=00001631-100507.rml
https://download.scidb.cn/download?fileId=62295e16d7561b594fb67f59&dataSetType=personal&username=36680@qq.com&fileName=00001547-100507.rml
https://download.scidb.cn/download?fileId=61a0ca3f89f14b48842ceef8&dataSetType=personal&username=36680@qq.com&fileName=00001169-100507.rml
https://download.scidb.cn/download?fileId=61a0ca3f89f14b48842cee8c&dataSetType=personal&username=36680@qq.com&fileName=00001222-100507.rml

file_name_list = []
with open('778740145531650048.txt', 'r') as file:
    for i in file:
        file_name_list.append(i.strip())

3. 下载和保存文件

# 逐行读取网址
processing_bar = tqdm.tqdm(file_name_list)
fault = []
for line in processing_bar:
    try:
        url = line.strip()  # 去除行末尾的换行符和空格
        # 文件名在fileName=后面
        filename = url.split('fileName=')[1]
        filename = unquote(filename)  # 解码
        folder = filename.split('.')[0].split('[')[0]
        # 创建文件夹
        if not os.path.exists(folder):
            os.makedirs(folder)
        
        
        if os.path.exists(os.path.join(folder, filename)):
            # 判断已经存在的文件是否大于4KB，如果大于则跳过
            if os.path.getsize(os.path.join(folder, filename)) > 1024 * 4:
            	# 切换processing_bar上面的文字
                processing_bar.set_postfix(info="Skip %s" % filename)
                continue
        
        
        processing_bar.set_postfix(info="Downloading %s" % filename)
        
        # 发起GET请求下载网址内容, 计时
        begin = time.time()
        response = requests.get(url)
        end = time.time()
        download_time = end - begin  # 下载总时间
        delay_time = random.randint(-10, 20) # 延迟时间波动范围 -10——20
        time.sleep(max(60 - download_time + delay_time, 0))  # 在60s上下波动
        
        # 保存下载内容到本地文件
        with open(os.path.join(folder, filename), 'wb') as output_file:
            output_file.write(response.content)
        
    except:
        fault.append(filename)

4. 结束并保存失败内容

print(fault)
save_fault = open('fault.txt', 'w')
for i in fault:
    save_fault.write(i+'\n')
save_fault.close()
        

完整代码

import requests
import os
from urllib.parse import unquote
import tqdm
import time
import random
# 打开文本文件

file_name_list = []
with open('778740145531650048.txt', 'r') as file:
    for i in file:
        file_name_list.append(i.strip())


# 逐行读取网址
processing_bar = tqdm.tqdm(file_name_list)
fault = []
for line in processing_bar:
    try:
        url = line.strip()  # 去除行末尾的换行符和空格
        # 文件名在fileName=后面
        filename = url.split('fileName=')[1]
        filename = unquote(filename)  # 解码
        folder = filename.split('.')[0].split('[')[0]
        # 创建文件夹
        if not os.path.exists(folder):
            os.makedirs(folder)
        
        
        if os.path.exists(os.path.join(folder, filename)):
            # 判断已经存在的文件是否大于10KB，如果大于则跳过
            if os.path.getsize(os.path.join(folder, filename)) > 1024 * 4:
                processing_bar.set_postfix(info="Skip %s" % filename)
                continue
        
        
        processing_bar.set_postfix(info="Downloading %s" % filename)
        
        # 发起GET请求下载网页内容, 计时
        begin = time.time()
        response = requests.get(url)
        end = time.time()
        download_time = end - begin
        delay_time = random.randint(-10, 20)
        time.sleep(max(60 - download_time + delay_time, 0))
        
        # 保存下载内容到本地文件
        with open(os.path.join(folder, filename), 'wb') as output_file:
            output_file.write(response.content)
        
    except:
        fault.append(filename)
print(fault)
save_fault = open('fault.txt', 'w')
for i in fault:
    save_fault.write(i+'\n')
save_fault.close()