Python Scrapy 多进程启动爬虫

对于需要多任务进行爬虫爬取时，使用爬虫命令爬取时没法进行连续进行爬取任务，使用如下程序实现多任务爬取

#引入你的爬虫
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
import time,os
import logging
from scrapy.utils.project import get_project_settings
import xlrd
from xlutils.copy import copy

#在控制台打印日志
configure_logging()
#CrawlerRunner获取settings.py里的设置信息
runner = CrawlerRunner(get_project_settings())

@defer.inlineCallbacks
def crawl():
    while True:

        logging.info("new cycle starting")
        while tasknum() > 0:

            print('peter正在执行任务')
            yield runner.crawl('peter')
    reactor.stop()
#判断Excel中是否还有要爬取的任务
def tasknum():
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = curPath[:curPath.find("cnkispider\\") + len("cnkispider\\")]  # 获取myProject，也就是项