下次试试用scrapy模块
# coding=utf-8
import requests
import threadpool
def download_file(i):
"""下载文件"""
download_url = f"https://www.txt2016.com/e/DownSys/xiazai/?classid=1&pathid=0&id={i}"
file = requests.get(download_url)
try:
filename = file.headers["Content-Disposition"][21:][:-19].encode("iso8859-1").decode("utf-8")
filename = filename.replace("/","-") + ".txt"
# print(filename)
with open(f"./txt/{filename}", "wb") as txt_file:
txt_file.write(file.content)
print(i, ": ", filename)
except KeyError:
print(file.headers)
print(file.status_code)
print("网址不存在")
if __name__ == "__main__":
urls = [i for i in range(2275, 113642)] # urls里任务很多, 比如几百个, 让pool自己去调度
pool = threadpool.ThreadPool(8) # 线程池设置,最多同时跑8个线程
tasks = threadpool.makeRequests(download_file, urls)
# makeRequests构造线程task请求, 第一个参数是线程函数, 第二个是参数数组
[pool.putRequest(task) for task in tasks]
# 列表推导式, putRequest向线程池里加task, 让pool自己去调度task
pool.wait() # 等所有任务结束