[jupyter] python3并行下载文件url

import gevent

from gevent import socket

from gevent import monkey; monkey.patch_all()

import sys

# import urllib2 py3弃用
import urllib.request

def download(url):

    try:

        url_opener = urllib.request.urlopen(url)

    except:

        print 'open url error'

        return

    if url_opener.code!=200:

        print 'return code is:%d'%(url_opener.code)

        return

    if not url_opener.headers.has_key('Content-Length'):

        print 'no content length'

        return

    content_length = long(url_opener.headers['Content-Length'])

    download_size = 0

    file_name = url[url.rfind('/')+1:]

    target_file = open(file_name, 'wb')

    while download_size<content_length:

        try:

            str_content = url_opener.read(1024)

        except Exception,e:

            print 'read error:%s' % (str(e))

            return

        if not str_content or len(str_content)==0:

            print 'read error, connection close'

            return

        target_file.write(str_content)

        download_size += len(str_content)

        print '[%s]:download size:%d, progress:%d' % (

                    file_name,

                    download_size,

                    int(float(download_size)/float(content_length)*100.0))

    print '[%s]:download complete!' % (file_name)



# if __name__=='__main__':

#     argc = len(sys.argv)

#     if argc<2:

#         print 'usage:%s <url> [url...]' % (sys.argv[0])

#         sys.exit(-1)

#     jobs = [gevent.spawn(download,url) for url in sys.argv[1:]]

#     gevent.joinall(jobs, timeout=600)