高德地图python爬虫 商家_Python爬虫练习：爬取高德地图地铁线路及站点数据

前言

本文的文字及图片来源于网络,仅供学习、交流使用,不具有任何商业用途,版权归原作者所有,如有问题请及时联系我们以作处理。

作者：Darcy频道

# coding=utf-8

import requests

import time

import json

import ast

import os

import utils

from lxml import etree

PAGE_URL = 'http://map.amap.com/subway/index.html?&1100'

DATA_URL = 'http://map.amap.com/service/subway?srhdata='

HEADER = {

"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"}

def fetchAllCity(url, header):

r = requests.get(url, header)

html = r.content

element = etree.HTML(html)

options = element.xpath("//a[contains(@class, 'city')]")

cities = []

for option in options:

city = {

'id': option.get('id'),

'name': option.get('cityname'),

'text': option.text

}

cities.append(city)

return cities

def parseCityData(citys):

lw = open('./lwkt.txt', 'w')

lw.write('wkt' + '\n')

pw = open('./pwkt.txt', 'w')

pw.write('wkt' + '\n')

for city in citys:

parseCityPointFromApi(city, lw, pw)

def parseCityPointFromApi(city, lw, pw):

url = DATA_URL + "{}_drw_{}.json".format(city['id'], city['name'])

print(url)

json_str = requests.get(url).text

res = json.loads(json_str)

res = res['l']

for r in res:

st = r['st']

coords = []

for s in st:

_coords = s.get('sl', '').split(',')

coords.append(_coords)

pwkt = '"POINT ({} {})"'.format(_coords[0], _coords[1])

pw.write(pwkt + '\n')

new_coords = ','.join(['%s %s' % (coord[0], coord[1]) for coord in coords])

lwkt = '"LINESTRING(%s)"' % new_coords

lw.write(lwkt + '\n')

def main():

cities = fetchAllCity(PAGE_URL, HEADER)

print(cities)

parseCityData(cities)

if __name__ == '__main__':

main()