python使用ThreadPoolExecutor线程池实现的生产者消费者模型

觉得有用的道友拿去用吧，支持获取线程任务的返回值，参考链接：https://www.jianshu.com/p/b9b3d66aa0be

不获取线程返回值的版本：

import os
import time
import redis
import pickle
from threading import Thread
from concurrent.futures import ThreadPoolExecutor

task_list_key = 'task_list_key'
redis_session = redis.Redis()
max_workers = (os.cpu_count() or 1) * 5


def push_task(task_list_key):  # 生产者
    task_id = 1
    while True:
        redis_session.rpush(task_list_key, pickle.dumps({'time': time.time(), 'task_id': task_id}))
        task_id += 1
        time.sleep(0.2)


def task_func(task_info):
    time.sleep(1)  # 任务处理时长
    print(pickle.loads(task_info))


def deal_task(task_list_key):  # 消费者
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        while True:
            task_info = redis_session.lpop(task_list_key)
            if task_info:
                executor.submit(task_func, task_info)
            else:
                time.sleep(0.1)  # 不加这句CPU会爆炸

Thread(target=push_task, args=(task_list_key,)).start()
Thread(target=deal_task, args=(task_list_key,)).start()

获取线程返回值的版本：

import os
import time
import redis
import pickle
from threading import Thread
from concurrent.futures import ThreadPoolExecutor, as_completed

task_list_key = 'task_list_key'
redis_session = redis.Redis()
max_workers = (os.cpu_count() or 1) * 5


def push_task(task_list_key):  # 生产者
    task_id = 1
    while True:
        redis_session.rpush(task_list_key, pickle.dumps({'time': time.time(), 'task_id': task_id}))
        task_id += 1
        time.sleep(0.2)


def task_func(task_info):
    time.sleep(1)  # 任务处理时长
    task_info = pickle.loads(task_info)
    # print(task_info)
    return task_info


def deal_task(task_list_key):  # 消费者
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        tasks = []
        while True:
            task_info = redis_session.lpop(task_list_key)
            if task_info:
                tasks.append(executor.submit(task_func, task_info))
            else:
                time.sleep(0.1)  # 不加这句CPU会爆炸
            if len(tasks) >= 5:
                futures = as_completed(tasks)  # 返回任务的对象，这里不耗时
                print([future.result() for future in futures])  # 耗时的任务真正阻塞的地方，注意取得的结果是乱序的
                tasks = []

Thread(target=push_task, args=(task_list_key,)).start()
Thread(target=deal_task, args=(task_list_key,)).start()