import scrapy
from scrapy.http import FormRequest
class CaijiaSpider(scrapy.Spider):
name = 'caijia'
#allowed_domains = ['http://www.xinfadi.com.cn/priceDetail.html']
def start_requests(self):
self.url = 'http://www.xinfadi.com.cn/getPriceData.html'
self.data = {'limit': '20',
'current': '1'}
self.MAX_PAGE = 17938
yield FormRequest(url=self.url,formdata=self.data,method='post')
def parse(self, response):
print('正在获取第' + str(int(self.data['current'])) + '页')
dic =response.json()
price_list = dic['list']
item = {}
for test in range(1, len(price_list)):
info = price_list[test]
item['fenlei'] = info.get('prodCat')
item['name'] = info.get('prodName')
item['low_price'] = info.get('lowPrice')
item['ave_price'] = info.get('avgPrice')
item['max_price'] = info.get('highPrice')
item['guige'] = info.get('specInfo')
item['place'] = info.get('place')
item['danwei'] = info.get('unitInfo')
item['data'] = info.get('pubDate')
yield item
#解析下一页数据
if int(self.data['current'])>= self.MAX_PAGE:return
self.data['current'] = str(int(self.data['current'])+1)
yield FormRequest(url=self.url,formdata=self.data,method='post')
setting设置
终端运行命令:scrapy crawl caijia -o xinfadi.csv爬取并保存为xinfadi.csv