python多线程下载编程_Python3 多线程下载代码

Python3 多线程下载代码,貌似原版源自Axel这个多线程下载工具。

'''

Created on 2014-10-24

@author: Maple

'''

import sys

import os

import time

import getopt

import urllib.request

import urllib.parse

from threading import Thread

#===============================================================================

# def download(url, output=os.getcwd(), blocks=6, proxies=local_proxies)

# output:输出文件路径,默认为当前路径

# blocks:线程数

# proxies:代理地址

#===============================================================================

local_proxies = {}#代理地址

class Maple(Thread):

version = "Mozilla/5.0"

def __init__(self, threadname, url, filename, ranges=0, proxies={}):

Thread.__init__(self, name=threadname)

self.name = threadname

self.url = url

self.proxies = proxies

self.filename = filename

self.ranges = ranges

self.downloaded = 0

def run(self):

try:

self.downloaded = os.path.getsize( self.filename ) #获取已下载的文件字节块块,支持断点续传

except OSError:

#print 'never downloaded'

self.downloaded = 0

opener=GetUrlOpener(self.proxies) #根据代理参数生成相应的url opener

if self.ranges: #ranges为线程需要下载的文件块的字节范围

# rebuild start poind

self.startpoint = self.ranges[0] + self.downloaded #从已下载字节块后的位置开始下载

# This part is completed

if self.startpoint >= self.ranges[1]:

self.downloaded = self.ranges[1] - self.ranges[0]

print ('Part %s has been downloaded over.' % self.filename)

return

opener.addheaders=[('Range','bytes={}-{}'.format(self.startpoint, self.ranges[1])),('User-agent','Mozilla/5.0')] #添加请求头部内容,仅下载指定范围的字节,伪装成浏览器请求

print ('task %s will download from %d to %d' % (self.name, self.startpoint+1, self.ranges[1]+1))

else: #ranges未指定(文件大小未知,无法切割),从已下载字节块后的位置开始下载剩余全部字节

self.startpoint = self.downloaded

opener.addheaders=[('Range','bytes={}-'.format(self.startpoint)),('User-agent','Mozilla/5.0')]

self.fetchsize = 16384 #每次读取的字节数

self.urlhandle = opener.open(self.url) #打开文件地址

data = self.urlhandle.read( self.fetchsize )

while data: #循环读取数据写入临时文件,并更新已下载字节数

filehandle = open( self.filename, 'ab+' )

filehandle.write( data )

filehandle.close()

self.downloaded += len( data )

data = self.urlhandle.read( self.fetchsize )

def Sec2Time(second): #将秒数转换为标准时间格式。以为有现成的函数,结果愣是没找到

day=second//(3600*24)

second-=day*3600*24

hour=second//3600

second -=hour*3600

minute=second//60

second-=minute*60

if day == 0:

if hour == 0:

if minute == 0:

return '{:0.2f}S.'.format(second)

else:

return '{:02}M:{:0.2f}S'.format(minute,second)

else:

return '{:02}H:{:02}M:{:0.2f}S'.format(hour,minute,second)

else:

return '{:03}D:{:02}H:{:02}M:{:0.2f}S'.format(day,hour,minute,second)

def GetUrlOpener(proxies={}): #分析代理参数,返回url opener。完整代理格式:user/passwd@http://127.0.0.1:8087。如格式不同,需要修改此分析函数

if proxies:

try:

ap=proxies.split('@')

if len(ap) > 1:

auth=ap[0]

addr=ap[1]

else:

addr=ap[0]

auth=''

if '://' in addr:

ptype=addr[:addr.find('://')]

phost=addr[addr.find('://')+3:]

else:

ptype='http'

phost=addr

proxy={ptype:ptype+'://'+phost}

proxy_handler = urllib.request.ProxyHandler(proxy)

except Exception as ex:

print(ex)

return urllib.request.build_opener()

try:

authlist=auth.split('/')

if len(authlist) > 1:

user=authlist[0]

passwd=authlist[1]

proxy_auth_handler = urllib.request.ProxyBasicAuthHandler()

proxy_auth_handler.add_password('realm',phost,user,passwd)

opener = urllib.request.build_opener(proxy_handler,proxy_auth_handler)

else:

opener = urllib.request.build_opener(proxy_handler)

return opener

except Exception as ex:

print(ex)

return urllib.request.build_opener(proxy_handler)

else:

# urlHandler=urllib.request.urlopen(url)

return urllib.request.build_opener()

def GetUrlFileInfo(url,proxies={}): #获取要下载的文件的信息,包括文件名,文件类型和文件大小

scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url) #分析url

filename=urllib.parse.unquote(path) #如果url中的文件名部分存在中文,将其正确解码出来

filename=filename.split('/')[-1]

opener=GetUrlOpener(proxies) #通过网络请求读取响应头部,根据头部获取文件信息。文件名以服务器返回的文件名信息为准

urlHandler=opener.open(url)

headers=urlHandler.info()

if 'Content-Disposition' in headers: #Content-Disposition字段有可能获取到文件名,不过可能是乱码,没找到解决办法

disposition=headers.get('Content-Disposition')

if 'filename=' in disposition:

filename = disposition.split('filename=')[1]

if filename[0] == '"' or filename[0] == "'":

filename = filename[1:-1]

filename=urllib.parse.unquote(filename)

if filename:

(name,ext)=os.path.splitext(filename)

else:

(name,ext)=('Unknown','')

if 'Content-Length' in headers: #获取文件长度,如果获取失败,则只能使用单线程下载

length=int(headers.get('Content-Length'))

else:

length=-1

(type, kind)=headers.get('Content-Type').split('/') #获取文件类型,备用

infos=[(name,ext),(type,kind),length]

return infos

def SpliteBlocks(totalsize, blocknumber): #根据指���的线程数参数和获取到的文件长度划分各线程的下载范围

blocksize = totalsize//blocknumber

ranges = []

for i in range(0, blocknumber-1):

ranges.append((i*blocksize, i*blocksize +blocksize - 1))

ranges.append(( blocksize*(blocknumber-1), totalsize -1 ))

return ranges

def islive(tasks): #检查各线程是否全部下载完成

for task in tasks:

if task.isAlive():

return True

return False

def download(url, target=os.getcwd(), blocks=6, proxies=local_proxies):

flag=True

print('Retrieving resource information...')

url=urllib.parse.quote(url,safe='/%&@=+?$;,:') #将提供的url编码,非英文字符将被编码为标准格式

try:

infos=GetUrlFileInfo(url,proxies) #获取文件信息

except Exception as ex:

print(ex)

flag=False

if flag: 

if not os.path.exists(target):

os.makedirs(target)

size=infos[2] #获取到的文件大小

output=os.path.join(target,''.join(infos[0])) #根据获取到的文件名和指定的保存目录生成完整路径

type=infos[1][0]

starttime=time.time() #开始计时

print('Infomation:')

print('FileName:{0} FileType:{1} FileLength:{2}'.format(''.join(infos[0]),'/'.join(infos[1]),infos[2] if int(infos[2]) > 0 else 'Unknown')) #打印获取到的文件信息

if size > 0: #size大于0表示成功获取文件长度,可以进行多线程下载

print('Starting multithread download...')

ranges = SpliteBlocks( size, blocks )

else: #只能单线程下载,线程数置1,ranges置空,

print('Starting single thread download...')

ranges=()

blocks=1

threadname = [ infos[0][0]+"_thread_%d" % i for i in range(0, blocks) ] #生成线程名

filename = [ infos[0][0]+ "_tmpfile_%d" % i for i in range(0, blocks) ] #生成各线程的临时文件名

tasks = []

for i in range(0,blocks): #生成下载线程,设置为后台线程后启动,将线程加入到线程列表中

task = Maple( threadname[i], url, filename[i], ranges[i] if ranges else ranges,proxies)

task.setDaemon( True )

task.start()

tasks.append( task )

time.sleep( 1 )

downloaded = 0

while islive(tasks): #统计线程列表中各线程的状态,输出下载进度

downloaded = sum( [task.downloaded for task in tasks] )

if size > 0:

process = downloaded/float(size)*100

show = '\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)

else:

show = '\rDownloaded:%d ' % downloaded

sys.stdout.write(show)

sys.stdout.flush()

time.sleep( 0.2 )

endtime=time.time() #下载完成后停止计时

consuming=Sec2Time(endtime-starttime)

if size > 0: #多线程下载的后续处理

downloadsize = 0

for i in filename:

downloadsize += os.path.getsize(i)

if downloadsize == size:

show = '\rFilesize:%d Downloaded:%d Completed:%.2f%%\n' % (size, downloadsize,100)

else:

show = '\nSize is not mathed!\n'

flag=False

else: #单线程下载的后续处理

show = '\nTotal Size: %d\n'% downloaded

sys.stdout.write(show)

sys.stdout.flush()

if flag: #确认下载的临时文件没问题后将各文件整合为最终的目标文件

print('Integrating files...')

num=1

while os.path.exists(output): #防止与本地已存在文件重名

fname,fext=os.path.splitext(output)

if '('+str(num-1)+')'+fext in output:

output = output.replace('('+str(num-1)+')'+fext,'('+str(num)+')'+fext)

else:

fname += '('+str(num)+')'

output = fname+fext

num +=1

if len(filename) ==1 : #单线程下载的话,直接将下载的文件重命名为目标文件即可

os.rename(filename[0], output)

else: #多线程临时文件整合

filehandle = open( output, 'wb+' )

for i in filename:

try:

f = open( i, 'rb' )

filehandle.write( f.read() )

f.close()

os.remove(i)

except Exception as ex:

print(ex)

filehandle.close()

if os.path.exists(output):

print('Download Complete!')

else:

print('Failed to generate target file!')

try:

#os.remove(output)

pass

except:

pass

else:

for i in filename:

try:

os.remove(i)

pass

except:

pass

print('Download Failed!')

pass

print('Consuming: {}\n'.format(consuming)) #输出耗时

    else:

        print('Failed to retrieve resource information!')

        sys.exit()

def main(argv): #处理传入参数,使用了getopt模块,另外有一个更强大的处理传入参数的模块optparse

try:

options,args=getopt.getopt(argv,'hu:f:n:p:',['help','url=','target=','num=','proxy='])

except Exception as ex:

print(ex)

sys.exit()

num = 2

url,target,proxies= '','',''

url = 'http://www.pygtk.org/dist/pygtk2-tut.pdf'

target = '/home/maple/Desktop'

#proxies = 'http://127.0.0.1:8087'

#proxies={}

for name, value in options:

if name in ('-h','--help'):

print('No Help ^^')

sys.exit()

if name in ('-u','--url'):

url = value

if name in ('-t','--target'):

target = value

if name in ('-n','--num'):

num = int(value)

if name in ('-p','--proxy'):

proxies=value

#check args

download(url,target,num,proxies)

if __name__ == '__main__':

main(sys.argv[1:])

这段代码在异常处理方面写得有些乱,没怎么关心异常处理,需要���再改吧

另外多线程下载时,如果使用了代理,会导致下载到的文件与服务器提供的文件大小不符。从而下载失败。我使用的是GoAgent,代理服务器会自动对目标文件进行多线程下载,无视程序指定的下载字节范围。第一个线程就会下载到完整的文件,其他线程会下载冗余内容。没有找到规范的处理办法。变通的处理办法有2种:

1、将使用了代理的多线程下载强制指定为单线程下载

2、不进行文件大小的校验,将多线程下载的0号临时文件重命名为目标文件,其他临时文件删除。

两种方法实现都很简单,但是破坏代码的整体逻辑。没有加入代码中。运行截图:

151213160580801.png

下面关于Python的文章您也可能喜欢,不妨看看:

Python 的详细介绍:请点这里

Python 的下载地址:请点这里

logo.gif

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值