import json
from klein import Klein
from scrapy import signals
import urllib
from scrapy.crawler import CrawlerRunner
import re
app = Klein()
class MyCrawlerRunner(CrawlerRunner):
def crawl(self, crawler_or_spidercls, *args, **kwargs):
# keep all items scraped
self.items = []
# create crawler (Same as in base CrawlerProcess)
crawler = self.create_crawler(crawler_or_spidercls)
# handle each item scraped
crawler.signals.connect(self.item_scraped, signals.item_scraped)
# create Twisted.Deferred launching crawl
dfd = self._crawl(crawler, *args, **kwargs)
# add callback - when crawl is done cal return_items
dfd.addCallback(self.return_items)
return dfd
def item_scraped(self, item, response, spider):
self.items.append(item)
def return_items(self, result):
return self.items
def return_spider_output(output):
print(output)
try:
return json.dumps([dict(item) for item in output][0])
except Exception as e:
print(e)
@app.route(’/api/labelsfashion/productlist/’,branch=True)
def labelsfashionlist(request):
url = request.uri.decode()
urls = re.findall(r’productlist/(.*)’,str(url))
urls = ‘’.join(urls)
urls = urls.split(’/’)
FromCurrencyCode = urls[0]
spiderurl = [urllib.parse.unquote(urls[1])]
page = urls[2]
runner = MyCrawlerRunner()
deferred = runner.crawl(LabelsFashionSpider.LabelsFashionListSpider, start_urls = spiderurl, FromCurrencyCode = FromCurrencyCode,page = page)
deferred.addCallback(return_spider_output)
return deferred
if name == ‘main’:
app.run(“0.0.0.0”, 8080)