使用scrapy构建新的FormRequest爬取新发地菜价

import scrapy
from scrapy.http import FormRequest

class CaijiaSpider(scrapy.Spider):
    name = 'caijia'
    #allowed_domains = ['http://www.xinfadi.com.cn/priceDetail.html']

    def start_requests(self):
        self.url = 'http://www.xinfadi.com.cn/getPriceData.html'
        self.data = {'limit': '20',
                'current': '1'}
        self.MAX_PAGE = 17938
        yield FormRequest(url=self.url,formdata=self.data,method='post')

    def parse(self, response):
        print('正在获取第' + str(int(self.data['current'])) + '页')
        dic =response.json()
        price_list = dic['list']
        item = {}
        for test in range(1, len(price_list)):
            info = price_list[test]
            item['fenlei'] = info.get('prodCat')
            item['name'] = info.get('prodName')
            item['low_price'] = info.get('lowPrice')
            item['ave_price'] = info.get('avgPrice')
            item['max_price'] = info.get('highPrice')
            item['guige'] =  info.get('specInfo')
            item['place'] =  info.get('place')
            item['danwei'] = info.get('unitInfo')
            item['data'] = info.get('pubDate')

            yield item

        #解析下一页数据

        if int(self.data['current'])>= self.MAX_PAGE:return

        self.data['current'] = str(int(self.data['current'])+1)
        yield FormRequest(url=self.url,formdata=self.data,method='post')

setting设置

 终端运行命令：scrapy crawl caijia -o xinfadi.csv爬取并保存为xinfadi.csv