使用reactor多线程运行scrapy

# -*- coding: utf-8 -*-
import threading
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.project import get_project_settings


runner = CrawlerRunner(get_project_settings())
@defer.inlineCallbacks
def crawl():
    while True:
        yield runner.crawl("scrapy项目中spider文件名")
    reactor.stop()

def do_job():
    crawl()
    reactor.run()

if __name__ == '__main__':
        thread = []
        for i in range(60):
            t = threading.Thread(target=do_job)
            t.daemon = True  # 设置线程daemon  主线程退出，daemon线程也会推出，即时正在运行
            t.start()
            thread.append(t)
        for j in thread:
            j.join()

 

代理如上图：执行该文件就可以多线程运行scrapy项目

该文件的路径如图：

文件存放在项目目录下