proxyee-down真的很不错,其实原理很简单。python估计也能实现。不过目前有一个需求就是批量下载,但是目前的版本(3.12)不支持。我知道它内部有个server,看了半天源码,不能找到具体的请求参数。于是另辟途径,运行下它的前端代码(vue实现),然后又捣鼓了半天,搞定!
直接上源码吧,不解释。(python 实现,依赖aiohttp)
如果觉得好,麻烦github点个星星(有那么重要吗:)代码地址:https://github.com/woshimanong1990/proxyee-down-batch-create-task
#coding:utf-8
import asyncio
import sys
import time
import os
import cgi
import re
import json
from urllib import parse
from urllib.parse import urlparse, unquote_plus
import aiohttp
async def fectch_config(session):
# 获取配置信息,只用到了部分信息
url = "http://127.0.0.1:26339/config"
header={
"Content-Type": "application/json;charset=UTF-8"
}
async with session.get(url, headers=header) as response:
if response.status // 100 != 2:
print("fectch_config, status", response.status)
return {}
config = await response.json()
config_keys = ["filePath","connections","timeout","retryCount","autoRename","speedLimit"]
new_config = {}
# 啰嗦,应该有简单的方法
for key in config_keys:
new_config[key] = config.get(key,None)
return new_config
async def get_head_info(session, url):
# 获取 request 和 response 信息
# {"request":{"method":"GET","url":"https://nodejs.org/dist/v8.12.0/node-v8.12.0-x64.msi",
# "heads":{"Host":"nodejs.org","Connection":"keep-alive","User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36",
# "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8","Referer":"nodejs.org"},"body":null},
# "response":{"fileName":"node-v8.12.0-x64.msi","totalSize":16445440,"supportRange":true}}
request_url = "http://127.0.0.1:26339/util/resolve"
data={
"method":"GET",
"url":url,
"heads":{},
"body":""
}
header={
"Accept":"application/json, text/plain, */*",
"Content-Type":"application/json;charset=UTF-8",
"Origin":"http://127.0.0.1:26339",
"Referer":"http://127.0.0.1:26339/",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36"
}
#print("get header data", data)
async with session.put(request_url, headers=header, json=data) as response:
if response.status // 100 != 2:
print("post_data error, status:{} reason{} ".format(response.status, await response.text()))
return {}
return await response.json()
async def post_data(session, data):
# 发送一个创建任务的请求到proxyee-down
url = "http://127.0.0.1:26339/tasks"
header={
"Accept":"application/json, text/plain, */*",
"Content-Type":"application/json;charset=UTF-8",
"Origin":"http://127.0.0.1:26339",
"Referer":"http://127.0.0.1:26339/",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36"
}
#print("post data", json.dumps(data))
async with session.post(url, headers=header, json=data) as response:
if response.status // 100 != 2:
print("post_data error,url:{} status:{} reason:{}".format(data.get("request",{}).get("url",""),response.status, await response.text()))
#print(response.request_info)
return None
return await response.text()
async def create_task(session, url):
# 创建任务,需要先发送两个请求,然后post,创建任务
config = await fectch_config(session)
request_response_data = await get_head_info(session, url)
if not config:
print("config is empty")
return
if not request_response_data:
print("request_response_data is empty")
return
data={
"config":config,
}
data.update(request_response_data)
return await post_data(session, data)
async def main(urls):
tasks = []
async with aiohttp.ClientSession() as session:
for url in urls:
task = create_task(session,url)
tasks.append(task)
await asyncio.gather(*tasks)
if __name__ == "__main__":
url = "https://nodejs.org/dist/v8.12.0/node-v8.12.0-x64.msi"
loop = asyncio.get_event_loop()
urls = [url] # 添加自己的下载链接
# 可以加载json的文件
# with open("download_urls.json", "r") as f:
# urls = json.load(f)
# loop.run_until_complete(asyncio.gather(*[main(url, 30) for url in urls[0:5]]))
loop.run_until_complete(main(urls))
补充:
建了一个qq群:389954854,主要目的是互帮互助,通过交流促进python技术。很多初学者往往不知道如何下手,还等什么,快快加入进来吧!
或者关注微信公众号:python码码有趣的