python多线程实现同时下载_Python实现多线程下载

#!/usr/bin/python#-*- coding: utf-8 -*-#filename: paxel.py

'''It is a multi-thread downloading tool

It was developed follow axel.

Author: volans

E-mail: volansw [at] gmail.com'''

importsysimportosimporttimeimporturllibfrom threading importThread

local_proxies= {'http': 'http://131.139.58.200:8080'}classAxelPython(Thread, urllib.FancyURLopener):'''Multi-thread downloading class.

run() is a vitural method of Thread.'''

def __init__(self, threadname, url, filename, ranges=0, proxies={}):

Thread.__init__(self, name=threadname)

urllib.FancyURLopener.__init__(self, proxies)

self.name=threadname

self.url=url

self.filename=filename

self.ranges=ranges

self.downloaded=0defrun(self):'''vertual function in Thread'''

try:

self.downloaded=os.path.getsize( self.filename )exceptOSError:#print 'never downloaded'

self.downloaded =0#rebuild start poind

self.startpoint = self.ranges[0] +self.downloaded#This part is completed

if self.startpoint >= self.ranges[1]:print 'Part %s has been downloaded over.' %self.filenamereturnself.oneTimeSize= 16384 #16kByte/time

print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])

self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))

self.urlhandle=self.open( self.url )

data=self.urlhandle.read( self.oneTimeSize )whiledata:

filehandle= open( self.filename, 'ab+')

filehandle.write( data )

filehandle.close()

self.downloaded+=len( data )#print "%s" % (self.name)

#progress = u'\r...'

data=self.urlhandle.read( self.oneTimeSize )def GetUrlFileSize(url, proxies={}):

urlHandler= urllib.urlopen( url, proxies=proxies )

headers=urlHandler.info().headers

length=0for header inheaders:if header.find('Length') != -1:

length= header.split(':')[-1].strip()

length=int(length)returnlengthdefSpliteBlocks(totalsize, blocknumber):

blocksize= totalsize/blocknumber

ranges=[]for i in range(0, blocknumber-1):

ranges.append((i*blocksize, i*blocksize +blocksize - 1))

ranges.append(( blocksize*(blocknumber-1), totalsize -1))returnrangesdefislive(tasks):for task intasks:iftask.isAlive():returnTruereturnFalsedef paxel(url, output, blocks=6, proxies=local_proxies):'''paxel'''size=GetUrlFileSize( url, proxies )

ranges=SpliteBlocks( size, blocks )

threadname= [ "thread_%d" % i for i inrange(0, blocks) ]

filename= [ "tmpfile_%d" % i for i inrange(0, blocks) ]

tasks=[]for i inrange(0,blocks):

task=AxelPython( threadname[i], url, filename[i], ranges[i] )

task.setDaemon( True )

task.start()

tasks.append( task )

time.sleep(2)whileislive(tasks):

downloaded= sum( [task.downloaded for task intasks] )

process= downloaded/float(size)*100show= u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' %(size, downloaded, process)

sys.stdout.write(show)

sys.stdout.flush()

time.sleep(0.5)

filehandle= open( output, 'wb+')for i infilename:

f= open( i, 'rb')

filehandle.write( f.read() )

f.close()try:

os.remove(i)pass

except:passfilehandle.close()if __name__ == '__main__':

url= "http://www.pygtk.org/dist/pygtk2-tut.pdf"output= 'pygtk2.pdf'paxel( url, output, blocks=4, proxies={} )