pyhton多线程调用scrapy框架

# -*- coding: utf-8 -*-
import threading
import os
from time import sleep


def crawl():
    os.system('scrapy crawl spider_name -s LOG_FILE=all.log')
    # 不想看到控制台打印debug信息 就加  -s LOG_FILE=all.log 【将debug信息接入all.log文件】


if __name__ == '__main__':
    while 1:
        thnums = threading.enumerate()
        if len(thnums) < 10:
            for i in range(60):
                t = threading.Thread(target=crawl)
                t.daemon = True
                t.start()
        elif len(thnums) < 55:
            for i in range(10):
                t = threading.Thread(target=crawl)
                t.daemon = True
                t.start()
        else:
            sleep(2)