Spider
继承 RedisSpider 的 Spider
import scrapy
from circ.items import CircItem
from copy import deepcopy
from scrapy_redis.spiders import RedisSpider
class DangSpider(RedisSpider):
name = 'dang'
allowed_domains = ['dangdang.com']
redis_key = 'dang'
def parse(self, response):
item = CircItem()
div_list = response.xpath('//div[@class="con flq_body"]/div')[2:-1]
for div in div_list:
for dl in div.xpath('.//div[@class="col eject_left"]/dl'):
book_class = dl.xpath('./dt/a/text()').extract()
item['book_class'] = book_class[0].strip() if book_class else None
for i in dl.xpath('./dd/a'):
item['book_label'] = i.xpath('./@title').extract()[0].strip()
href = i.xpath('./@href').extract()[