python-利用queue和threading实现多线程下载

主代码：

#----------------------------------------------------
    #download worker
    #----------------------------------------------------
    queue_download = Queue.Queue(0)
    DOWNLOAD_WORKERS = 20
    for i in range(DOWNLOAD_WORKERS):
        DownloadWorker(queue_download).start()  #start a download worker
        
    for md5 in MD5S:
        queue_download.put(md5)
    for i in range(DOWNLOAD_WORKERS):
        queue_download.put(None)

downloadworkers.py

类继承 threading.Thread,重载run方法..在__init__中调用threading.Thread.__init__(self),

在run方法中实现耗时的操作

import threading
import Queue
import md5query
import DOM
import os,sys
########################################################################
class DownloadWorker(threading.Thread):
    """"""

    #----------------------------------------------------------------------
    def __init__(self, queue):
        """Constructor"""
        self.__queue = queue
        threading.Thread.__init__(self)


    def run(self):
        while 1:
            md5 = self.__queue.get()
            if md5 is None:
                break #reached end of queue
            #this is a time-cost produce
            self._down(md5)

            print "task:", md5, "finished"

    def _down(self, md5):
        config = {
            'input':sys.stdin, 
            'output':'./samples', 
            'location':'xxx', 
            'has-fn':False, 
            'options':{'connect.timeout':60, 'timeout':3600}, 
            'log':file('logs.txt', 'w'), 
        }
        print 'download %s...' % (md5)
        try:
            data = downloadproc(config['location'], config['options'])  我的下载过程
            if data:
                dom, fileData = md5query.splited(data)
                filename = md5
                if config['has-fn']:
                    filename = '%s_%s' % (md5, dom.nodeValue2('xxxxxxx', '').encode('utf-8'))   这是我的下载的方法
                f = file(os.path.join(config['output'], filename), 'w')
                f.write(fileData)
                f.close()

                print '%s\tok' % (md5)
            else:
                print>>config['log'], '%s\t%s' % (md5, 'failed')
        except Exception, e:
            print>>config['log'], '%s\t%s' % (md5, str(e))