python gevent下载_python: gevent轻松实现并行下载多个文件

#!/bin/python

# file: download.py

import gevent

from gevent import socket

from gevent import monkey; monkey.patch_all()

import sys

import urllib2

def download(url):

try:

url_opener = urllib2.urlopen(url)

except:

print 'open url error'

return

if url_opener.code!=200:

print 'return code is:%d'%(url_opener.code)

return

if not url_opener.headers.has_key('Content-Length'):

print 'no content length'

return

content_length = long(url_opener.headers['Content-Length'])

download_size = 0

file_name = url[url.rfind('/')+1:]

target_file = open(file_name, 'wb')

while download_size

try:

str_content = url_opener.read(1024)

except Exception,e:

print 'read error:%s' % (str(e))

return

if not str_content or len(str_content)==0:

print 'read error, connection close'

return

target_file.write(str_content)

download_size += len(str_content)

print '[%s]:download size:%d, progress:%d' % (

file_name,

download_size,

int(float(download_size)/float(content_length)*100.0))

print '[%s]:download complete!' % (file_name)

if __name__=='__main__':

argc = len(sys.argv)

if argc<2:

print 'usage:%s [url...]' % (sys.argv[0])

sys.exit(-1)

jobs = [gevent.spawn(download,url) for url in sys.argv[1:]]

gevent.joinall(jobs, timeout=600)

'''

感谢Thijs同学的推荐！利用协程，在一个函数中从头写到尾就能实现并行，实在是太爽了。

相比之下，twisted中眼花缭乱的回调，学习曲线实在是太陡峭了！

'''

阅读全文

类别：Python 查看评论