(29)爬虫小说实例

main
from scrapy.cmdline import execute

execute('scrapy crawl zw'.split())

zw.py
import scrapy


class ZwSpider(scrapy.Spider):
    name = 'zw'
    allowed_domains = ['zwdu.com']
    start_urls = ['https://www.zwdu.com/book/32934/13022999.html']

    def parse(self, response):
        title = response.xpath('//h1/text()').extract_first()
        content = ''.join(response.xpath('''//div[@id='content']/text()''').extract()).replace('    ','\n    ')
        yield {'title':title,
               'content':content}
        next_url = response.xpath('''//div[@class='bottem1']/a[3]/@href''').extract_first()
        base_url = 'https://www.zwdu.com{}'.format(next_url)
        yield scrapy.Request(base_url,callback = self.parse)

pipelines.py
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter


class ZwwPipeline():
    def open_spider(self,spider):
        self.file = open('三寸人间.txt','w',encoding='utf-8')

    def process_item(self, item, spider):
        title = item['title']
        content = item['content']
        info = ''.join(title + '\n' + content + '\n')
        self.file.write(info)
        return item

    def file_close(self,spider):
        self.file.close()

setting.py
# Scrapy settings for zww project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'zww'

SPIDER_MODULES = ['zww.spiders']
NEWSPIDER_MODULE = 'zww.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 2
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'zww.middlewares.ZwwSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'zww.middlewares.ZwwDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'zww.pipelines.ZwwPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

其爬取成功文件：

这里是引用
写在连载前

亲爱的读者大大们，说心里话，我现在心里既激动，又难过。
激动的是，你们又可以登上我的小车，在我这个粉嫩帅气00后司机的行驶中，带你们去体验故事里的跌宕起伏，惊心动魄。
至于难过的是……我的假期没了……我的心好痛，需要安慰。
不过我也休息够了，感觉全身上下都精力充沛，充满了码字的力量，很多时候一想到新书，就不吐不快。
更是想念我最最最最亲爱的读者小哥哥，读者小姐姐们，来，抱一下吧。
然后再向大家汇报一下这几个月我的生活，假期这段时间，我是拼了老命减肥，现在看到镜子里的自己，我都觉得，天啊，此人好帅，他是谁……
咳咳，你们想知道我的独门减肥秘诀么？
同时，这段日子，我在休息之余，也在完善新书的思路，新书对我来说，是一次很大很大的挑战，我始终在考虑，仙侠小说如何能开出不同的花朵。
我曾经在之前的多本书里，加入了很多元素，也取得了一些成绩，可心里总是觉得，有些不大满意，我认为仙侠小说，是可以写出更好，更精彩，甚至不同背景的故事。
背景这两个字，请大家划重点……
现在已经准备很充分了，这一次的新书，我觉得很满意，尤其是故事里的主人公，你们明天就会知道，他是如何的与众不同……
也会明白，我为了这本书，付出了多大的努力与辛酸，现在想想，为了写这本书，我真是拼了老命啊。
友情提醒，明天开始，出现高能，请大家系好安全带……
我们，明天中午12点，见！
最后，一念永恒的外传，我之前是发在了公众微信号里，有读者没看过，我在下面发一下。
差点忘了喊一句，求收藏啊！！！！减肥秘诀换收藏！！！！！
####
岁月悠悠，一晃而过。
永恒灵界星光璀璨，一百零八万族群繁荣昌盛，一代代强者辈出，太古也好，主宰也罢，纷纷如雨后春笋一般，在各自的时代里，成为明亮的而星辰。
尤其是永恒仙域，更是这无尽星辰中最特殊，也是最明亮的一颗。
白小纯的故事，已经化作了传说，随着多年后他带着亲人朋友离开了永恒仙域，他曾经的往事，在这永恒灵界内已然成为了神话，哪怕多年过去，也依旧时常别人提起，那一段时光的辉煌，似乎从某种程度上，就已经代表了永恒灵界巅峰的历史。
没有人知道白小纯去了什么地方，只能从一些当年魁皇朝留下的典籍里找到一些蛛丝马迹，似乎当年