这里主要给出spider中的代码
wxjcspider.py
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from wxjc.items import WxjcItem
class WxjcspiderSpider(CrawlSpider):
name = 'wxjcspider'
allowed_domains = ['wxapp-union.com']
start_urls = ['http://www.wxapp-union.com/portal.php?mod=list&catid=2&page=1']
rules = (
#这里我有一个问题,当我给出完整的url allow=r' http://www.wxapp-union.com/portal.php? mod=list&catid=2&page=\d+'
#时,爬虫只能爬取第一页的数据
Rule(LinkExtractor(allow=r'.+mod=list&catid=2&page=\d+'),follow=True),
Rule(LinkExtractor(allow=r'http://www.wxapp-union.com/article-\d+-1\.html'),callback="parse_item",follow=False)
)
def parse_item(self, response):
content = response.xpath("//td[@id='article_content']//text()").getall()
title = response.xpath("//div[@class='cl']/h1[@class='ph']/text()").get()
out_time = response.xpath("//span[@class='time']/text()").get().strip()
content = ''.join(content).strip()
item = WxjcItem(content=content,title =title,out_time=out_time)
yield item
from scrapy.exporters import JsonLinesItemExporter
class WxjcPipeline(object):
def process_item(self, item, spider):
with open('wxjc.json','ab') as fp:
JsonLinesItemExporter(fp,encoding='utf-8').export_item(item)
#这里的return item一定不能删了,在这个类中可能有多个pipline,需要item
return item