上一篇博客说过如何使用高德开放平台,那么现在来试试使用它的接口来获取信息把!
# -*- coding: utf-8 -*-
import scrapy
import json
class GaodeSpider(scrapy.Spider):
name = 'gaode'
allowed_domains = ['amap.com']
start_urls = ['http://amap.com/']
def parse(self, response):
# 构造URL列表
locations = ['蜀山区', '包河区', '庐阳区', '瑶海区', '新站高新区', '肥西区', '肥东区', '巢湖市', '长丰县', '庐江县']
# 先搜索了合肥有哪几个大的区,然后获取这几个就行
for location in locations:
for page in range(1, 40):
url = 'https://restapi.amap.com/v3/place/text?keywords={}小区&city=340103&offset=20' \
'&page={}&key="这里是在高德开放平台获取的key"&citylimit=true'.format(location, page)
yield scrapy.Request(
url=url,
callback=self.parse_data,
)
def parse_data(self, response):
item = {}
datas = json.loads(response.text)
for data in datas['pois']:
item['adname'] = data['adname']
item['name'] = data['name']
item['loca'] = data['location']
yield item