scrapy当当当当 连衣裙分类

scrapy startproject dangdang
scrapy genspider -t basic cao dangdang.com
# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class DangdangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #name 表示商品名称
    name = scrapy.Field()
    # 商品价格
    price = scrapy.Field()
    # 商品链接地址
    link = scrapy.Field()
# -*- coding: utf-8 -*-
import scrapy
from dangdang.items import DangdangItem
from scrapy.http import Request
import json


class CaoSpider(scrapy.Spider):
    name = "cao"
    allowed_domains = ["dangdang.com"]
    start_urls = (
        'http://category.dangdang.com/pg1-cid4008149.html',
    )

    def parse(self, response):
        item = DangdangItem()
        item['name'] = response.xpath("//p[@class='name']/a[@name='itemlist-title']/text()").extract()
        item['price'] = response.xpath("//p[@class='price']/span[@class='price_n']/text()").extract()
        item['link'] = response.xpath('//a[@class="pic"]/@href').extract()



        # 提取完成后返回item
        yield item

        for i in range(1,10):
            url = "http://category.dangdang.com/pg"+str(i)+"-cid4008149.html"
            yield Request(url,callback=self.parse)

        for i in range(100):
            print(item['name'][i]+'----' +item['price'][i] + '---' + item['link'][i])
pipelines.py 存储本地json文件
# 修改settings.py

ITEM_PIPELINES = {
    'dangdang.pipelines.DangdangPipeline': 300,
}
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json

class DangdangPipeline(object):
    def __init__(self):
        self.file = codecs.open("./data.json",'wb',encoding="utf-8")

    def process_item(self, item, spider):
        i = json.dumps(dict(item),ensure_ascii=False)

        line = i + '\n'
        self.file.write(line)

        return item

    def close_spider(self,spider):
        self.file.close()
  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值