python 并行_Python中运行并行任务技巧

# -*- coding: utf8 -*-

import os

import time

import Queue

import threading

from PIL import Image

def create_thumbnail(filename, size=(128, 128)):

try:

fp, fmt = filename.rsplit('.', 1)

im = Image.open(filename)

im.thumbnail(size, Image.ANTIALIAS)

im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)

return '%s thumbnail success!' % filename

except Exception:

return '%s thumbnail failed!' % filename

def get_image_paths(folder):

return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]

class Consumer(threading.Thread):

def __init__(self, queue):

threading.Thread.__init__(self)

self._queue = queue

def run(self):

while True:

content = self._queue.get()

if isinstance(content, str) and content == 'quit':

break

respone = create_thumbnail(content)

print 'Bye bye!'

def Producer():

filenames = get_image_paths('images')

queue = Queue.Queue()

worker_threads = build_worker_pool(queue, 4)

start_time = time.time()

for filename in filenames:

queue.put(filename)

for worker in worker_threads:

queue.put('quit')

for worker in worker_threads:

worker.join()

print time.time() - start_time

def build_worker_pool(queue, size):

workers = []

for _ in range(size):

worker = Consumer(queue)

worker.start()

workers.append(worker)

return workers

if __name__ == '__main__':

Producer()