scrapy当当当当 连衣裙分类

scrapy startproject dangdang
scrapy genspider -t basic cao dangdang.com

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class DangdangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #name 表示商品名称
    name = scrapy.Field()
    # 商品价格
    price = scrapy.Field()
    # 商品链接地址
    link = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from dangdang.items import DangdangItem
from scrapy.http import Request
import json


class CaoSpider(scrapy.Spider):
    name = "cao"
    allowed_domains = ["dangdang.com"]
    start_urls = (
        'http://category.dangdang.com/pg1-cid4008149.html',
    )

    def parse(self, response):
        item = DangdangItem()
        item['name'] = response.xpath("//p[@class='name']/a[@name='itemlist-title']/text()").extract()
        item['price'] = response.xpath("//p[@class='price']/span[@class='price_n']/text()").extract()
        item['link'] = response.xpath('//a[@class="pic"]/@href').extract()



        # 提取完成后返回item
        yield item

        for i in range(1,10):
            url = "http://category.dangdang.com/pg"+str(i)+"-cid4008149.html"
            yield Request(url,callback=self.parse)

        for i in range(100):
            print(item['name'][i]+'----' +item['price'][i] + '---' + item['link'][i])

pipelines.py 存储本地json文件

# 修改settings.py

ITEM_PIPELINES = {
    'dangdang.pipelines.DangdangPipeline': 300,
}

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json

class DangdangPipeline(object):
    def __init__(self):
        self.file = codecs.open("./data.json",'wb',encoding="utf-8")

    def process_item(self, item, spider):
        i = json.dumps(dict(item),ensure_ascii=False)

        line = i + '\n'
        self.file.write(line)

        return item

    def close_spider(self,spider):
        self.file.close()