抓取代码:
抓取后,可以用 Beyond Compare 4 工具查看比较数据抓取是否正确。
package com.lc.iot.controller.collection;
import com.alibaba.fastjson.JSON;
import com.lc.iot.domain.map.MapItemModel;
import com.lc.iot.domain.map.MapModel;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@Slf4j
@RestController
@RequestMapping(value = "/geoArea")
public class GeoAreaController {
@GetMapping("/GetArea")
public String GetArea() throws IOException {
String countryUrl = "https://geo.datav.aliyun.com/areas_v3/bound/geojson?code=100000_full";
String localCountryPath = "F:\\tools\\map\\" + "100000" + ".json";
MapModel countryModel = null;
if (!new File(localCountryPath).exists()) {
String countryResult = new RestTemplate().getForObject(countryUrl, String.class);
FileUtils.write(new File(localCountryPath), countryResult, "UTF-8", false);
countryModel = JSON.parseObject(countryResult, MapModel.class);
} else {
String countryResult = FileUtils.readFileToString(new File(localCountryPath), StandardCharsets.UTF_8);
countryModel = JSON.parseObject(countryResult, MapModel.class);
}
log.info(String.format("%s请求成功", countryUrl));
for (MapItemModel provinceItem : countryModel.features) {
String provinceAdcode = provinceItem.properties.adcode;
String provinceName = provinceItem.properties.name;
Integer provinceChildrenNum = provinceItem.properties.childrenNum;
if (provinceChildrenNum == null) {
break;
}
String provinceUrl = "https://geo.datav.aliyun.com/areas_v3/bound/geojson?code=" + provinceAdcode + (provinceChildrenNum > 0 ? "_full" : "");
try {
String localProvincePath = "F:\\tools\\map\\" + provinceAdcode + ".json";
MapModel provinceModel = new MapModel();
if (!new File(localProvincePath).exists()) {
String provinceResult = new RestTemplate().getForObject(provinceUrl, String.class);
FileUtils.write(new File(localProvincePath), provinceResult, "UTF-8", false);
provinceModel = JSON.parseObject(provinceResult, MapModel.class);
} else {
String provinceResult = FileUtils.readFileToString(new File(localProvincePath), StandardCharsets.UTF_8);
provinceModel = JSON.parseObject(provinceResult, MapModel.class);
}
for (MapItemModel cityItem : provinceModel.features) {
String cityAdcode = cityItem.properties.adcode;
String cityName = cityItem.properties.name;
Integer cityChildrenNum = cityItem.properties.childrenNum;
if (cityChildrenNum == null) {
break;
}
String cityUrl = "https://geo.datav.aliyun.com/areas_v3/bound/geojson?code=" + cityAdcode + (cityChildrenNum > 0 ? "_full" : "");
try {
String localCityPath = "F:\\tools\\map\\" + cityAdcode + ".json";
MapModel cityModel = new MapModel();
if (!new File(localCityPath).exists()) {
String cityResult = new RestTemplate().getForObject(cityUrl, String.class);
FileUtils.write(new File(localCityPath), cityResult, "UTF-8", false);
cityModel = JSON.parseObject(cityResult, MapModel.class);
} else {
String cityResult = FileUtils.readFileToString(new File(localCityPath), StandardCharsets.UTF_8);
cityModel = JSON.parseObject(cityResult, MapModel.class);
}
for (MapItemModel townItem : cityModel.features) {
String townAdcode = townItem.properties.adcode;
String townName = townItem.properties.name;
Integer townChildrenNum = townItem.properties.childrenNum;
try {
String localTownPath = "F:\\tools\\map\\" + townAdcode + ".json";
if (!new File(localTownPath).exists()) {
if (townChildrenNum == null) {
break;
}
String townUrl = "https://geo.datav.aliyun.com/areas_v3/bound/geojson?code=" + townAdcode + (townChildrenNum > 0 ? "_full" : "");
String townResult = new RestTemplate().getForObject(townUrl, String.class);
FileUtils.write(new File(localTownPath), townResult, "UTF-8", false);
MapModel townModel = JSON.parseObject(townResult, MapModel.class);
}
} catch (Exception ex) {
log.info(String.format("请求%s - %s数据失败", provinceName + ">>" + cityName + ">>" + townName, cityUrl));
}
}
} catch (Exception cityEx) {
log.info(String.format("请求%s - %s数据失败", provinceName + ">>" + cityName, cityUrl));
}
}
} catch (Exception provinceEx) {
log.info(String.format("请求%s - %s数据失败", provinceName, provinceUrl));
}
}
return "";
}
}