学习scrapy，起点小说简易爬虫

# -*- coding: utf-8 -*-
import scrapy


class QidianSpider(scrapy.Spider):
    name = 'qidian'
    allowed_domains = ['qidian.com']
    start_urls = ['https://read.qidian.com/chapter/ZOJ_pWRoTg9wKI0S3HHgow2/42hiyQwiuVS2uJcMpdsVgA2',]

    def parse(self, response):
        
        chapter_name = response.xpath('//span[@class="content-wrap"]/text()').extract_first()
        chapter_txt = response.xpath('//div[@class="read-content j_readContent"]/p/text()').extract()
        
        next_url = response.xpath('//a[@id="j_chapterNext"]/@href').extract_first()
        yield {
            'chapter_name':chapter_name,
            'chapter_txt':chapter_txt
        }
        yield scrapy.Request(response.urljoin(next_url),callback=self.parse)