scrapy crawlspider爬取小程序社区教程文章

这里主要给出spider中的代码
wxjcspider.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from wxjc.items import WxjcItem


class WxjcspiderSpider(CrawlSpider):
    name = 'wxjcspider'
    allowed_domains = ['wxapp-union.com']
    start_urls = ['http://www.wxapp-union.com/portal.php?mod=list&catid=2&page=1']

    rules = (
    #这里我有一个问题，当我给出完整的url     allow=r' http://www.wxapp-union.com/portal.php? mod=list&catid=2&page=\d+'
    #时，爬虫只能爬取第一页的数据
        Rule(LinkExtractor(allow=r'.+mod=list&catid=2&page=\d+'),follow=True),
        Rule(LinkExtractor(allow=r'http://www.wxapp-union.com/article-\d+-1\.html'),callback="parse_item",follow=False)
            )

    def parse_item(self, response):
        content = response.xpath("//td[@id='article_content']//text()").getall()
        title = response.xpath("//div[@class='cl']/h1[@class='ph']/text()").get()
        out_time = response.xpath("//span[@class='time']/text()").get().strip()

        content = ''.join(content).strip()
        item = WxjcItem(content=content,title =title,out_time=out_time)
        yield item

pipelines.py

from scrapy.exporters import JsonLinesItemExporter

class WxjcPipeline(object):
    def process_item(self, item, spider):
        with open('wxjc.json','ab') as fp:
            JsonLinesItemExporter(fp,encoding='utf-8').export_item(item)
       #这里的return item一定不能删了，在这个类中可能有多个pipline，需要item
        return item