jsp调用python脚本_Python实现从脚本里运行scrapy的方法

本文实例讲述了Python实现从脚本里运行scrapy的方法。分享给大家供大家参考。具体如下：

#!/usr/bin/python

import os

os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'project.settings') #Must be at the top before other imports

from scrapy import log, signals, project

from scrapy.xlib.pydispatch import dispatcher

from scrapy.conf import settings

from scrapy.crawler import CrawlerProcess

from multiprocessing import Process, Queue

class CrawlerScript():

def __init__(self):

self.crawler = CrawlerProcess(settings)

if not hasattr(project, 'crawler'):

self.crawler.install()

self.crawler.configure()

self.items = []

dispatcher.connect(self._item_passed, signals.item_passed)

def _item_passed(self, item):

self.items.append(item)

def _crawl(self, queue, spider_name):

spider = self.craw