Python 代码到这个水平就行了！！

python pycurl实现断点续传和多线程下载

# -*- coding: utf8 -*-

import sys, os
import time, logging
import urllib, urlparse
import codecs, traceback
from tempfile import *
from dict4ini import DictIni
import pycurl

try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO

if os.name == ‘posix’:
# 使用pycurl.NOSIGNAL选项时忽略信号SIGPIPE
import signal
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
del signal

# 支持的协议
VALIDPROTOCOL = (‘http’, ‘ftp’)
# HTTP状态码
STATUS_OK = (200, 203, 206)
STATUS_ERROR = range(400, 600)
# 最小数据片大小(128kb)
MINPIECESIZE = 131072
# 最大连接数
MAXCONCOUNT = 10
# 最大重试数
MAXRETRYCOUNT = 5
# 日志级别
LOGLEVEL = logging.DEBUG
# 清屏命令
CLS = ‘cls’ if os.name == ‘nt’ else ‘clear’

# 下载日志文件
DLOG = ‘download.log’

def Traceback():
    s = StringIO()
    traceback.print_exc(file=s)
    return s.getvalue()

class Connection:
    def __init__(self, url):
        self.curl = pycurl.Curl()
        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
        self.curl.setopt(pycurl.MAXREDIRS, 5)
        self.curl.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.curl.setopt(pycurl.TIMEOUT, 300)
        self.curl.setopt(pycurl.NOSIGNAL, 1)
        self.curl.setopt(pycurl.WRITEFUNCTION, self.write_cb)
        self.curl.setopt(pycurl.URL, url)
        self.curl.connection = self

        # 合计下载字节数
        self.total_downloaded = 0

    def start(self, result, piece):
        if isinstance(piece, list):
            self.id = piece[0]
            self.name = ‘Piece%02d’ % piece[0]
            self.curl.setopt(pycurl.RANGE, ‘%d-%d’ % (piece[1], piece[2])) #设置断点范围
            self.piece_size = piece[2] – piece[1] + 1
            self.piece = piece
        else:
            self.id = 0
            self.name = ‘TASK’
            self.piece_size = piece
            self.piece = None

        # 一次连接的已下载字节数
        self.link_downloaded = 0
        # 一个片断的已下载字节数
        self.piece_downloaded = 0
        # 连接重试数
        self.retried = 0
        # 下载中止标志
        self.is_stop = False
        # 结果输出文件对象
        self.result = result
        self.piece = piece

    def retry(self):
        self.curl.setopt(pycurl.RANGE, ‘%d-%d’ % (self.piece[1] +
            self.piece_downloaded, self.piece[2]))
        if self.link_downloaded: # 上次连接中有数据返回？
            self.link_downloaded = 0
        else:
            self.retried += 1

    def close(self):
        self.curl.close()

    def write_cb(self, data):
        if self.piece:
            self.result.seek(self.piece[1] + self.piece_downloaded, 0)
            self.result.write(data)
            self.result.flush()
            size = len(data)
            self.link_downloaded += size
            self.piece_downloaded += size
            self.total_downloaded += size
        if self.is_stop: return -1

#FastDownload
class FastDownload:
    def __init__(self):
        file(DLOG, ‘w’)
        logging.basicConfig(level=LOGLEVEL,
            format=’[%(asctime)s][%(levelname)s] %(message)s’,
            filename=’download.log’,
            filenmode=’w')

        self.mcurl = pycurl.CurlMulti()

    def execute(self, url):
        ”’
        下载接口
        ”’
        self.url_info = self.url_check(url)
        if self.url_info:
            print ‘Download %s, Size %d’ % (self.url_info['file'],
            self.url_info['size'])
            self.pieces = self.make_pieces()
            self.allocate_space()
            self.download()

    # ***************************************************************

    def url_check(self, url):
        ”’
        下载地址检查
        ”’
        url_info = {}
        proto = urlparse.urlparse(url)[0]
        if proto not in VALIDPROTOCOL:
            print ‘Valid protocol should be http or ftp, but %s found<%s>!’ % (proto, url)
        else:
        ss = StringIO()
        curl = pycurl.Curl()
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.CONNECTTIMEOUT, 30)
        curl.setopt(pycurl.TIMEOUT, 300)
        curl.setopt(pycurl.NOSIGNAL, 1)
        curl.setopt(pycurl.NOPROGRESS, 1)
        curl.setopt(pycurl.NOBODY, 1)
        curl.setopt(pycurl.HEADERFUNCTION, ss.write)
        curl.setopt(pycurl.URL, url)

        try:
            curl.perform()
        except:
            pass

        if curl.errstr() == ” and curl.getinfo(pycurl.RESPONSE_CODE) in STATUS_OK:
            url_info['url'] = curl.getinfo(pycurl.EFFECTIVE_URL)
            url_info['file'] = os.path.split(url_info['url'])[1]
            url_info['size'] = int(curl.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
            url_info['partible'] = (ss.getvalue().find(‘Accept-Ranges’) != -1)

        return url_info

    def make_pieces(self):
        ”’
        分段信息生成
        ”’
        if self.url_info['partible']:
            file_size = self.url_info['size']
            num = MAXCONCOUNT
            while num * MINPIECESIZE > file_size and num > 1:
                num -= 1
                piece_size = int(round(self.url_info['size'] * 1.0 / num + 0.5))
                pieces = [[i, i * piece_size, (i + 1) * piece_size - 1] for i in
                range(num)]
                pieces[-1][2] = self.url_info['size'] – 1
        else:
            pieces = [self.url_info['size']]
        return pieces

    def allocate_space(self):
        ”’
        预分配文件空间(通用？)
        ”’
        afile = file(self.url_info['file'], ‘wb’)
        afile.truncate(self.url_info['size'])
        afile.close()

    # ***************************************************************

    def show_progress(self, downloaded, elapsed):
        ”’
        显示下载进度
        ”’
        percent = min(100, downloaded * 100.0 / self.url_info['size'])
        if elapsed == 0:
            rate = 0
        else:
            rate = downloaded * 1.0 / 1024.0 / elapsed
            info = ‘ D/L: %d/%d (%6.2f%%) – Avg: %4.1fkB/s’ % (downloaded,self.url_info['size'], percent, rate)
            space = ‘ ‘ * (60 – len(info))

            prog_len = int(percent * 20 / 100)
            prog = ‘|’ + ‘o’ * prog_len + ‘.’ * (20 – prog_len) + ‘|’

            sys.stdout.write(info + space + prog)
            sys.stdout.flush()
            sys.stdout.write(‘\b’ * 82)

    def close_connection(self, c):
        ”’
        关闭连接
        ”’
        self.connections.remove(c)
        c.close()

    def process_curl(self, curl):
        ”’
        下载结果处理
        ”’
        self.mcurl.remove_handle(curl)
        c = curl.connection
        c.errno = curl.errno
        c.errmsg= curl.errmsg
        self.working_connections.remove(c)
        if c.errno == pycurl.E_OK:
            c.code = curl.getinfo(pycurl.RESPONSE_CODE)
            d = self.process_ok(c)
        else:
            d = self.process_error(c)
        return d

    def process_ok(self, c):
        ”’
        下载成功处理
        ”’
        if c.code in STATUS_OK:
            assert c.piece_downloaded == c.piece_size
            msg = ‘%s: Download successed’ % c.name
            logging.info(msg)
            msg = ‘%s: Download %s out of %d’ % (c.name, c.piece_downloaded,
            c.piece_size)
            logging.debug(msg)
            self.free_connections.append(c)
            elif c.code in STATUS_ERROR:
            msg = ‘%s: Error<%d>! Connection will be closed’ % (c.name,
            c.code)
            logging.warning(msg)
            self.close_connection(c)
            self.pieces.append(c.piece)
        else:
            raise Exception(‘%s: Unhandled http status code %d’ % (c.name,c.code))

    def process_error(self, c):
        ”’
        下载失败处理
        ”’
        msg = ‘%s: Download failed<%s>’ % (c.name, c.errmsg)
        logging.error(msg)
        if self.url_info['partible'] and c.retried < MAXRETRYCOUNT:
            c.retry()
            self.working_connections.append(c)
            self.mcurl.add_handle(c.curl)
            msg = ‘%s: Try again’ % c.name
            logging.warning(msg)
        else:
            raise Exception(‘Download abort~~’)

    def download(self):
        ”’
        下载主过程
        ”’
        self.result = file(self.url_info['file'], ‘r+b’)
        self.connections = []
        for i in range(len(self.pieces)):
            c = Connection(self.url_info['url'])
            self.connections.append(c)
            self.free_connections = self.connections[:]
            self.working_connections = []

        ok = True
        start_time = time.time()
        try:
            while 1:
                while self.pieces and self.free_connections:
                    p = self.pieces.pop(0)
                    c = self.free_connections.pop(0)
                    c.start(self.result, p)
                    self.working_connections.append(c)
                    self.mcurl.add_handle(c.curl)
                    msg = ‘%s: Start downloading’ % c.name
                    logging.debug(msg)

        while 1:
            ret, handles_num = self.mcurl.perform()
            if ret != pycurl.E_CALL_MULTI_PERFORM: break

        while 1:
            queue_num, ok_list, err_list = self.mcurl.info_read()
        for curl in ok_list:
            curl.errno = pycurl.E_OK
            curl.errmsg = ”
            self.process_curl(curl)
        for curl, errno, errmsg in err_list:
            curl.errno = errno
            curl.errmsg = errmsg
            self.process_curl(curl)
        if queue_num == 0: break

        elapsed = time.time() – start_time
        downloaded = sum([c.total_downloaded for c in
        self.connections])
        self.show_progress(downloaded, elapsed)

        if not self.working_connections: break

        self.mcurl.select(1.0)
        except:
        logging.error(‘Error:’ + Traceback())
        ok = False
        finally:
        for c in self.connections:
            c.close()
            self.mcurl.close()

        if ok:
        msg = ‘Download Successed! Total Elapsed %ds’ % elapsed
        else:
        msg = ‘Download Failed!’
        print ‘\n’, msg
        logging.info(msg)

    if __name__ == ‘__main__’:
        os.system(CLS)

        if len(sys.argv) > 1:
            url = sys.argv[1]
        else:
            url = ‘http://www.python.org/ftp/python/2.5.2/python-2.5.2.msi’
            fd = FastDownload()
            fd.execute(url)

将这个上传来的, 目前在服务器内存中的这个文件保存下来, 使用Topic model 中ImageField的自带方法

from your.models import Topic
topic_image = request.FILES['topic_image']

topic = Topic(title='tttttt')
img = topic.image.save('imagename.jpg', topic_image, save = False)
topic.save()

注意: 上面的img = topic.image.save('imagename.jpg', topic_image, save = False)就完成了图片文件的保存.

实际上它调用了 storage来保存这个文件.

而storage在保存文件的时候, 调用了 Storage.save(name, content)函数, 然后Storage.save(name, content)函数, 又调用了Storage._save(name, content). 注意下划线.

这里的name是保存到服务器上的文件名(可以理解为路径), content 参数是一个File类的实例, 不能仅仅是一个文件的指针fp, 或者是文件的内容(fp.read()), 而是对文件句柄和内容都进行包装之后的File类的实例.

因为 Storage.save(name, content)最终必须要调用 Storage.chunks() 这个函数, 但是fp文件指针和, fp.read()文件内容都没有这个方法,导致最后调用失败.

不过, Django提供了很多个File类:

django\core\files\base.py 中定义了 File, ContentFile
django\core\files\uploadedfile.py 中定义了 UploadedFile, TemporaryUploadedFile, InMemoryUploadedFile, SimpleUploadedFile

继承关系是:

File                                File 基类
----ContentFile                     继承自 File
----UploadFile                      继承自 File, 主要用于Form表单来保存文件, 一个Form.save()就可以搞定了.
--------TemporaryUploadedFile       继承自 UploadFile, 这个是用于保存文件到临时文件夹
--------InMemoryUploadedFile        继承自 UploadFile, 这个是把上传的文件保存到内存中, 在保存到硬盘之前, 可以进行其他操作
------------SimpleUploadedFile      继承自 InMemoryUploadedFile, 默认为简单的文本型数据上传, 使用StringIO

仅仅在view中, 使用model的某个FileField 来保存上传来的文件话, 如果是从request.FILES中获取的话, 就是InMemoryUploadedFile.

InMemoryUploadedFile的chunks() 实际上是假的, 因为既然读取到内存了, 那么就不用再像处理上传500M文件那样, 接收一段(chunk) 保存一段了, 因为它已经上传到服务器的内存了, 然后可以直接从内存中保存到服务器硬盘了.

当然, 这个保存到服务器也和django的最大上传文件的大小设置有关系, 如果 max_size 是2M的话, 那么只要文件小于2M, 就会直接使用InMemoryUploadedFile 来保存文件, 如果文件大小超过了 max_size, 那么就会变成一个使用chunk来接收文件的File类, 这里主要要参考 UploadFileHandler中的处理.

由于我没看源码, 这里可以猜测它为  TemporaryUploadedFile(相当于写入临时文件, 然后在复制, 移动到需要保存的位置)

也就是说, 用户上传文件

如果 文件大于上传最大的文件(这个肯定和服务器内存容量有关系, 你肯定不希望用户上传2G的文件一下把你的内存全部占用, 可能你会匀出2M内存来处理小文件)

----> 使用 TemporaryUploadedFile, 一边接收文件流, 接收一段就写入临时文件, 直到全部接收完毕

如果文件 小于上传限制(比如2M)

----> 直接使用内存来保存文件 即, InMemoryUploadedFile, 这个时候, 根本不需要操作什么 chunks, 但是还是提供了chunks函数, 只不过直接返回了整个文件的范围, 不像TemporaryUploadedFile, 它是返回一段段的数据, 数据大小是django中配置的最大chunk值.

好, 现在厘清了 TemporaryUploadedFile 和 InMemoryUploadedFile 的关系, 简单说, 超大文件使用TemporaryUploadedFile, 小文件使用 InMemoryUploadedFile.

那么上面的例子, 一个话题有个图片, 这个图片是用来指示话题的, 让人一看到图片就知道这个话题是和什么相关, 因此使用的是小图片.

它的保存, 仅仅使用了 ImageField.save(name, content), 就完成了, 即:

topic.image.save(name, content) # 这里的content, 实际上即是上面说的 InMemoryUploadedFile 的一个实例.

而这里的topic.image, 就是 ImageField 的实例.

调用过程:

1.ImageField.save(name, content)
2.---->Storage.save(name, content)
3.-------->Storage._save(name, content)
4.------------->name 会变成 Storage.get_avaiable_name(name), # 如果name已存在, 在name后边加下划线变成 name_.jpg
5.------------->Storage.chunks(name, content)

这几个过程中, content 都是指的 File 之类的实例, 这里是 InMemoryUploadedFile.

未完待续...