Effective Python 读书笔记: 第39条: 用Queue来协调各线程之间的工作

# -*- encoding: utf-8 -*-

from collections import deque
from queue import Queue
from threading import Lock
from threading import Thread
from time import sleep

'''
第39条: 用Queue来协调各线程之间的工作

关键:
1 管线将处理流程划分为几个节点，用多个线程同时执行任务
2 要注意防止某个阶段先入等待中
3 Queue类具备阻塞式操作，指定队列大笑，支持join方法
4 样例:
    downloadQueue = ClosableQueue()
    resizeQueue = ClosableQueue()
    uploadQueue = ClosableQueue()
    doneQueue = ClosableQueue()
    threads = [
        StoppableWorker(download, downloadQueue, resizeQueue),
        StoppableWorker(resize, resizeQueue, uploadQueue),
        StoppableWorker(upload, uploadQueue, doneQueue)
    ]
    for thread in threads:
        thread.start()
    for _ in range(1000):
        downloadQueue.put(object())
    downloadQueue.close()
    downloadQueue.join()

参考:
Effectiv Python 编写高质量Python代码的59个有效方法
'''

class MyQueue(object):
    def __init__(self):
        self.items = deque()
        self.lock = Lock()

    def put(self, item):
        with self.lock:
            self.items.append(item)

    def get(self):
        with self.lock:
            return self.items.popleft()


'''
作用:
用线程表示管线的各个阶段，Worker线程会从队列中取出任务，并对该任务
运行相关函数，把运行结果放到另一个队列中
'''
class Worker(Thread):
    def __init__(self, func, inQueue, outQueue):
        super(Worker, self).__init__()
        self.func = func
        self.inQueue = inQueue
        self.outQueue = outQueue
        self.polledCount = 0
        self.workDone = 0

    def run(self):
        while True:
            self.polledCount += 1
            try:
                item = self.inQueue.get()
            except IndexError:
                sleep(0.1)
            else:
                result = self.func(item)
                self.outQueue.put(result)
                self.workDone += 1

def download(value):
    pass


def resize(value):
    pass


def upload(value):
    pass


def task():
    downloadQueue = MyQueue()
    resizeQueue = MyQueue()
    uploadQueue = MyQueue()
    doneQueue = MyQueue()
    threads = [
        Worker(download, downloadQueue, resizeQueue),
        Worker(resize, resizeQueue, uploadQueue),
        Worker(upload, uploadQueue, doneQueue)
    ]
    for thread in threads:
        thread.start()
    for _ in range(1000):
        downloadQueue.put(object())
    for thread in threads:
        thread.join()
    # while len(doneQueue.items) < 1000:
    #     a = 1


queue = Queue()

def consumer2():
    print "Consumer waiting"
    queue.get()
    print "Consumer done"


def consumer():
    sleep(0.1)
    queue.get()
    print "Consumer got 1"
    queue.get()
    print "Consumer got 2"


def queueTask():
    thread = Thread(target=consumer)
    thread.start()


def queueTask2():
    thread = Thread(target=consumer)
    thread.start()
    print "Producer putting"
    queue.put(object())
    thread.join()
    print "Producer done"


class ClosableQueue(Queue):
    FINISH = object()

    def close(self):
        self.put(self.FINISH)

    def __iter__(self):
        while True:
            item = self.get()
            try:
                if item is self.FINISH:
                    return
                yield item
            finally:
                self.task_done()


class StoppableWorker(Thread):
    def __init__(self, func, inQueue, outQueue):
        super(StoppableWorker, self).__init__()
        self.func = func
        self.inQueue = inQueue
        self.outQueue = outQueue
        self.polledCount = 0
        self.workDone = 0

    def run(self):
        for item in self.inQueue:
            result = self.func(item)
            self.outQueue.put(result)


def betterQueueTask():
    downloadQueue = ClosableQueue()
    resizeQueue = ClosableQueue()
    uploadQueue = ClosableQueue()
    doneQueue = ClosableQueue()
    threads = [
        StoppableWorker(download, downloadQueue, resizeQueue),
        StoppableWorker(resize, resizeQueue, uploadQueue),
        StoppableWorker(upload, uploadQueue, doneQueue)
    ]
    for thread in threads:
        thread.start()
    for _ in range(1000):
        downloadQueue.put(object())
    downloadQueue.close()
    downloadQueue.join()
    resizeQueue.close()
    resizeQueue.join()
    uploadQueue.close()
    uploadQueue.join()
    print "doneQueue.qsize: {value} items finished".format(
        value=doneQueue.qsize()
    )
    # for thread in threads:
    #     thread.join()


def process():
    # task()
    betterQueueTask()


if __name__ == "__main__":
    process()