scrapy解决管道阻塞问题采用threadpool库线程池+twisted同步语法异步编程

实现方法：process_item和download任务函数像下面编写即可，其他管道像往常一样写法

import time
import threadpool
import random
from twisted.internet import defer

class VideoPipeline:
    def __init__(self):
        self.pool = threadpool.ThreadPool(10)  # 设置线程数
        pass

    def process_item(self, item, spider):
        """
        :param item:
        :param spider:
        :return:
        """

        finish_flag = random.randint(0,9)
        item['finish_flag'] = finish_flag
        # 线程池异步处理
        defer_item = defer.Deferred()
        task = threadpool.WorkRequest(self.download, args=(item,spider,finish_flag,defer_item,))
        self.pool.putRequest(task)
        return defer_item


    def download(self,item,spider,finish_flag,defer_item):
        """相当于长作业"""
        time.sleep(random.randint(1,10))
        if not finish_flag:
            defer_item.callback(dict())
        defer_item.callback(item)

    def process_item2(self, item):
        """ VideoPipeline 下一个优先级管道类方法"""
        print(item)

if __name__ == '__main__':
    a = VideoPipeline()
    for i in range(10):
        defer_item = a.process_item({},None)
        defer_item.addCallback(a.process_item2)
        print(11111111111)
    time.sleep(50)