Python学习笔记_5初识Scrapy框架（实现抓取股票行情）

stock.py

# -*- coding: utf-8 -*-
import scrapy
from items import StockstarItem,StockstarItemloader
class StockSpider(scrapy.Spider):
    name = 'stock'#定义爬虫名称
    allowed_domains = ['quote.stockstar.com']#定义爬虫域
    start_urls = ['http://quote.stockstar.com/stock/ranklist_a_3_1_1.html']#定义开始爬虫链接
    def parse(self, response):#撰写爬虫逻辑
     page=int(response.url.split("_")[-1].split(".")[0])#抓取页码
     item_nodes = response.css('#datalist tr')
     for item_node in item_nodes:# 根据item文件中定义的字段内容，进行字段内容抓取
        item_loader = StockstarItemloader(item=StockstarItem(),selector=item_node)
        item_loader.add_css("code","td:nth-child(1) a::text")
        item_loader.add_css("abbr","td:nth-child(2) a::text")
        item_loader.add_css("last_trade","td:nth-child(3) span::text")
        item_loader.add_css("chg_ratio","td:nth-child(4) span::text")
        item_loader.add_css("chg_amt","td:nth-child(5) span::text")
        item_loader.add_css("chg_ratio_5min", "td:nth-child(6) span::text")
        item_loader.add_css("volume", "td:nth-child(7)::text")
        item_loader.add_css("turn_over", "td:nth-child(8)::text")
        stock_item = item_loader.load_item()#保存json信息
        yield stock_item
     if item_nodes:
                next_page=page+1
                next_url=response.url.replace("{0}.html".format(page), "{0}.html".format(next_page))
                yield scrapy.Request(url=next_url, callback=self.parse)
                pass

itmes.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst
class StockstarItemloader(ItemLoader):
#自定义itemloader，用于存储爬虫所抓取的字段内容
 default_output_processor = TakeFirst()
class StockstarItem(scrapy.Item):#建立相应的字段
    # define the fields for your item here like:
    #name = scrapy.Field()
    code = scrapy.Field()#股票代码
    abbr = scrapy.Field()#股票简称
    last_trade = scrapy.Field()#最新价
    chg_ratio = scrapy.Field()#涨跌幅
    chg_amt = scrapy.Field()#涨跌额
    chg_ratio_5min = scrapy.Field()#5分钟涨幅
    volume = scrapy.Field()#成交量
    turn_over = scrapy.Field()#成交额
    pass

为解决在json文件中出现/Uxxx编码错误 采用在settings文件中附加

FEED_EXPORT_ENCODING = 'utf-8'代码解决