爬取当当图书信息

# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy


class DangSpider(scrapy.Spider):
    name = 'dang'
    allowed_domains = ['dangdang.com']
    start_urls = ['http://book.dangdang.com/']

    def parse(self, response):
        div_list=response.xpath('//div[@class="con flq_body"]/div')
        for div in div_list:
            item={}
            item['b_case']=div.xpath('./dl/dt//text()').extract()
            try:
                item['b_case']=[i.strip() for i in item['b_case'] if len(i.strip())>0][0]
            except:
                pass
            dl_list=div.xpath('.//dl[@class="inner_dl"]')
            for dl in dl_list:
                item['m_case']=dl.xpath('./dt//text()').extract()
                try:

                    item['m_case']=[i.strip() for i in item['m_case'] if len(i.strip())>0][0]
                except:
                    pass

                dd_list=dl.xpath('./dd/a')
                for dd in dd_list:
                    item['s_case']=dd.xpath('./@title').extract_first()
                    item['url'] = dd.xpath('./@href').extract_first()

                    yield scrapy.Request(url=item['url'],#有一个url就打印一次
                                         callback=self.parse_one_page,
                                         meta={'item':deepcopy(item)}

                        )
    def parse_one_page(self,response):
        item=response.meta.get('item')
        li_list=response.xpath('//ul[@class="list_aa "]/li')
        for li in li_list:
            item['title']=li.xpath('./a/@title').extract_first()
            item['img']=li.xpath('./a/img/@data-original').extract()
            item['price']=li.xpath('./p[2]//text()').extract()
            item['price']=''.join(item['price'])
            print(item)