一、item的编写
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class AutopjtItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
#定义好name用来存储商品名
name=scrapy.Field()
#定义好price用来存储商品价格
price=scrapy.Field()
#定义好link用来存储商品链接
link=scrapy.Field()
#定义好comnum用来存储商品评论数
comnum=scrapy.Field()
二、pipeline的编写
# -*- coding: utf-8 -*-
import codecs
import json
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
class AutopjtPipeline(object):
def __init__(self):
#打开mydata.json文件
self.file = codecs.open("F:/mydata3.json", "wb", encoding="utf-8")
def process_item(self, item, spider):
# item=dict(item)
# print(len(item["name"]))
# 每一页中包含多个商品信息,所以可以通过循环,每一次处理一个商品
#