安装scrapy-Redis

redis把数据保存在内存

MongoDB把数据保存在硬盘

pip install scrapy-redis

easy_install scrapy-redis

或者下载安装包下载。

scrapy 配置redis，在settings.py文件中配置redis

默认端口6379

#-*-coding:utf8-*-

from scrapy_redis.spiders import RedisSpider
from scrapy.selector import Selector
from scrapy.http import Request
from novelspider.items import NovelspiderItem
import re

class novSpider(RedisSpider):
    name = "novspider"
    redis_key = 'nvospider:start_urls'
    start_urls = ['http://www.daomubiji.com/'
                  #'http://www.daomubiji.com/qi-xing-lu-wang-01.html'
                  ]

    def parse(self,response):
        selector = Selector(response)
        table = selector.xpath('//table')