该代码实现了一个多进程下载文件的程序，通过读取url.txt文件获取URL列表，用户输入进程数量，程序会显示下载速度、已下载大小、总大小等信息。同时，有一个函数将文件大小转换为人类可读的形式。程序使用了requests库进行HTTP请求，multiprocessing和threading库进行多进程和线程管理。
no lock
from ctypes.wintypes import DOUBLE
from multiprocessing import Pool
import os, time, random
from ftplib import FTP
import datetime
#from ftplibnsf import FTP
import os, time
import multiprocessing
import gc
import threading
import sys
import random, requests
import traceback
SUFFIXES = {1000: ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
1024 : [ 'KiB' , 'MiB' , 'GiB' , 'TiB' , 'PiB' , 'EiB' , 'ZiB' , 'YiB' ]}
def approximate_size ( size , a_kilobyte_is_1024_bytes = True ):
'''Convert a file size to human-readable form.
Keyword arguments:
size -- file size in bytes
a_kilobyte_is_1024_bytes -- if True (default), use multiples of 1024
if False, use multiples of 1000
Returns: string
'''
if size < 0 :
raise ValueError ( 'approximate_size: number must be non-negative' )
multiple = 1024 if a_kilobyte_is_1024_bytes else 1000
for suffix in SUFFIXES [ multiple ]:
size /= multiple
if size < multiple :
return '{0:.1f} {1}' . format ( size , suffix )
raise ValueError ( 'approximate_size: number too large' )
urllist=[]
def getUrlList():
with open('url.txt','r') as f:
for line in f:
urllist.append(list(line.strip('\n').split(',')))
print(urllist)
#lock = threading.RLock()
lastTime = datetime.datetime.now()
lasttotal =0
#global nump
nump = 0
#def printdata(data):
# print(datetime.datetime.now()," ", len(data))
#total = 0
def getnump():
while True:
ten=input("请输入进程数量（1-99）:")
try:
x=eval(ten)
if (type(x)==int) and (x in range(0, 100)):
break
except:
pass
return x
def getcontent_length(content_length):
try:
x=eval(content_length)
if (type(x)==int) :
return x
except:
return 0
return 0
speed = 0
starttime = datetime.datetime.now()
def worker(urllist, i, l, totalOfTotal, nump):
def printdata(data, content_length, total, url, l, totalOfTotal, nump ):
lendata = len(data)
#puttotalOfTotal(totalOfTotal)
#global total
#global lock
global lastTime, lasttotal, speed, starttime
#global nump
#l.acquire()
try:
#total = total + lendata
content_length = max(content_length, 1)
thistime = datetime.datetime.now()
#interval = int(max(nump/2, 1))
interval = int(max(nump, 1))
#print('intervalllllllllllllllllllllllllllllllllllllll',nump, interval)
total_seconds = max(1, (thistime - lastTime ).total_seconds())
start_seconds = max(1, (thistime - starttime ).total_seconds())
if start_seconds < interval :
speed = (total- lasttotal)/start_seconds
if ( (thistime - lastTime ).total_seconds() > interval):
speed = (total- lasttotal)/interval
print(strpid, thistime," ", lendata, " ",
approximate_size(speed),'/S',start_seconds, 'seconds',
approximate_size(content_length), 'percent: {:.2%}'.format(total/content_length), url,
approximate_size(lasttotal), approximate_size(total), approximate_size(totalOfTotal.value))
if (speed > 600000):
print('=============================================')
lastTime = thistime
lasttotal = total
except Exception as e:
print('printdata:', url, str(e))
#finally:
# 修改完成，释放锁
#l.release()
total = 0
try:
#while 1:
global uplimit
# if totalOfTotal.value >uplimit:
# break
port = 21
#ftp = FTP() # 设置变量
timeout = 22
#total = 0
strpid = "进程号" + str(os.getpid()) + ' '+ str(i)+' '+ multiprocessing.current_process().name + ': '
#global urllist
rlist = random.randint(1,len(urllist))-1
rurl = urllist[rlist]
print(strpid, 'random:', rlist, ' ', rurl )
#print(strpid,": connecting ", ts_ip)
#print(strpid, ftp.connect(ts_ip, port, timeout))
#ftp.login(user, pwd)
#ftp.cwd("")
#localname = "CPE_D_" + str(msg) + ".rar"
#print("进程号:" + str(os.getpid()) + " 开始下载 " + localname)
total_seconds = 0
try:
#f = open(localname, 'wb')
filename = 'RETR ' + "1G.rar"
#print(rurl, rurl[0])
#r = requests.get( url='https://releases.ubuntu.com/20.04/ubuntu-20.04.3-desktop-amd64.iso', stream=True )
r = requests.get( url=rurl[0], stream=True )
print(strpid, rurl, r.headers, r.headers.get('content-length') )
content_length = 0
content_length = getcontent_length(r.headers.get('content-length'))
total = 0
global lasttotal, starttime
lasttotal = 0
starttime = datetime.datetime.now()
total_seconds = 0
for chunk in r.iter_content(chunk_size=1024):
if chunk:
total = total + len(chunk)
totalOfTotal.value = totalOfTotal.value + len(chunk)
printdata(chunk,content_length, total, rurl, l, totalOfTotal, nump)
global speed
total_seconds = max((datetime.datetime.now()- starttime).total_seconds(), 1)
#speed = int(total/total_seconds)
if (total_seconds>22) and(total>10000000) and (speed < 600000):
break
if (total_seconds>55) and (speed < 100000):
break
if total>content_length:
break
if totalOfTotal.value >uplimit:
break
#ftp.retrbinary(filename, f.write)
#ftp.retrbinary(filename, printdata)
#f.close() # 关闭文件
except Exception as e:
print(e.args)
print(strpid, rurl, str(e))
print(repr(e))
printe(e)
#print("进程号:" + str(os.getpid()) + " 下载中断 " )
#f = open(localname, 'rb')
#f.close()
#print("进程号:" + str(os.getpid()) + " 中断处理完成 " + localname)
# print("进程号:" + str(os.getpid()) + " 下载文件完成:" + localname)
time.sleep(3)
print("进程号:" + str(os.getpid()) + " done!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! " , 'speed:',
approximate_size(speed),'/S',approximate_size(total),total_seconds,'seconds', rurl)
# print(localname)
except Exception as e:
print('printdata:', rurl, str(e))
printe(e)
#f=open('error.txt', 'a+')
#f.write(str(datetime.datetime.now())+"进程号:" + str(os.getpid())+' '+'speed:'+
# approximate_size(speed)+'/S '+approximate_size(total)+str(total_seconds) +'seconds'+ rurl)
#f.close
def puttotalOfTotal(totalOfTotal):
#totalOfTotal = 0
global totalOfTotalname
with open(totalOfTotalname,'w') as f:
f.write(str(totalOfTotal.value))
totalOfTotalname = 'totalOfTotal.txt'
def gettotalOfTotal():
#totalOfTotal = 0
global totalOfTotalname
if os.path.exists(totalOfTotalname):
with open(totalOfTotalname,'r') as f:
s = f.read()
print('s', s)
#i = int(s)
#print('i', i)
return(s)
return '0'
#terminate() close()
def restartprocess(plist, totalOfTotal):
nump = len(plist)
for p in multiprocessing.active_children() :
#p = multiprocessing.Process(tp)
print('restarting ', p)
#p.close()
p.terminate()
time.sleep(1)
print('terminate ', p)
p.close()
gc.collect()
totalOfTotal1 = multiprocessing.Value('d', 0)
totalOfTotal1.value = float(gettotalOfTotal())
startprocess(nump, plist, totalOfTotal1)
return totalOfTotal1
#p.start()
#if not p.is_alive():
# p.start()
def printe(e):
print( sys._getframe().f_lineno, 'str(e):\t\t', str(e))
print( sys._getframe().f_lineno, 'repr(e):\t', repr(e))
#print( sys._getframe().f_lineno, 'e.message:\t', e.message)
print( sys._getframe().f_lineno, 'traceback.print_exc():', traceback.print_exc())
print( sys._getframe().f_lineno, 'traceback.format_exc():\n%s' % traceback.format_exc())
def startprocess(nump, plist, totalOfTotal):
plist.clear()
#totalOfTotal = multiprocessing.Value('d', 0)
#totalOfTotal.value = float(gettotalOfTotal())
for i in range(nump):
p = multiprocessing.Process(target=worker, args=(urllist,i, lock, totalOfTotal, nump))
p.start()
print('start ', p)
plist.append(p)
time.sleep(1)
gc.collect()
uplimit = 8400123456789
import ctypes
if __name__ == '__main__':
# while 1:
try:
multiprocessing.freeze_support()
getUrlList()
if os.path.exists(os.getcwd()):
s_list = os.listdir(os.getcwd())
for s_list_line in s_list:
if "CPE_D_" in s_list_line:
# print(os.getcwd() +"\\" +s_list_line)
os.remove(os.getcwd() + "\\" + s_list_line)
nump = getnump()
#po = Pool(nump) # 最大的进程数
#for i in range(0, 10000000000):
# po.apply_async(worker, (urllist, ))
# po.map(worker, (i,))
plist = []
lock = multiprocessing.Lock()
totalOfTotal = multiprocessing.Value('d', 0)
totalOfTotal.value = float(gettotalOfTotal())
print('gettotalOfTotal()', totalOfTotal.value)
for i in range(nump):
p = multiprocessing.Process(target=worker, args=(urllist,i, lock, totalOfTotal, nump))
p.start()
plist.append(p)
gc.collect()
stimes = 0
ttimes =0
lasttotalOfTotal = totalOfTotal.value
while 1:
try:
time.sleep(0.5)
stimes = stimes + 1
ttimes = ttimes + 1
puttotalOfTotal(totalOfTotal)
listactive_children = multiprocessing.active_children()
active_children = len( listactive_children )
if (ttimes % 22) == 0 :
print('multiprocessing.active_childrennnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn()', datetime.datetime.now(), lasttotalOfTotal,
totalOfTotal.value, ttimes, stimes, active_children )
# if (stimes > nump) or (lasttotalOfTotal == totalOfTotal.value) :
if (stimes > max(22, nump)) :
#restartprocess(plist)
stimes = 0
if (lasttotalOfTotal == totalOfTotal.value) :
#p = multiprocessing.Process(target=worker, args=(urllist,stimes, lock, totalOfTotal, nump))
#p.start()
restartprocess(plist, totalOfTotal)
lasttotalOfTotal = totalOfTotal.value
#if lasttotalOfTotal == totalOfTotal.value :
if ttimes > 4444:
print('ctypes.addressof( totalOfTotal)', ctypes.addressof( totalOfTotal._obj) )
print('ctypes.addressof( totalOfTotal)', totalOfTotal.__dict__.items() )
totalOfTotal = restartprocess(plist, totalOfTotal)
print('ctypes.addressof( totalOfTotal)', totalOfTotal.__dict__.items() )
print('ctypes.addressof( totalOfTotal)', ctypes.addressof( totalOfTotal._obj) )
ttimes = 0
if active_children < nump :
print('active_children < nump :active_children < nump :active_children < nump :')
p = multiprocessing.Process(target=worker, args=(urllist,stimes, lock, totalOfTotal, nump))
p.start()
# plist.append(p)
gc.collect()
if totalOfTotal.value >uplimit:
break
except Exception as e:
print('__main__ pool loop:', str(e))
printe(e)
time.sleep(2)
po.close()
po.join()
except Exception as e:
print('__main__', str(e))
printe(e)
time.sleep(2)
# pyinstaller -F ftp_NR_CPE_B.py
扫一扫
1761
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
