rides-scrapy 分布式爬取当当网图书

Spider

继承 RedisSpider 的 Spider

# -*- coding: utf-8 -*-
import scrapy
from circ.items import CircItem
from copy import deepcopy
from scrapy_redis.spiders import RedisSpider

class DangSpider(RedisSpider):
    name = 'dang'
    allowed_domains = ['dangdang.com']
    # start_urls = ['http://book.dangdang.com/']
    # redis 数据库内建立 lpush 表 redis_key 为表名 start_urls 为内容
    redis_key = 'dang'

    def parse(self, response):
        item = CircItem()
        div_list = response.xpath('//div[@class="con flq_body"]/div')[2:-1]
        for div in div_list:
            for dl in div.xpath('.//div[@class="col eject_left"]/dl'):
                book_class = dl.xpath('./dt/a/text()').extract()
                item['book_class'] = book_class[0].strip() if book_class else None
                for i in dl.xpath('./dd/a'):
                    item['book_label'] = i.xpath('./@title').extract()[0].strip()
                    href = i.xpath('./@href').extract()[