直接上代码:
//@SpringBootTest
class DemoApplicationTests {
@Test
void contextLoads() throws IOException {
RestTemplate restTemplate = new RestTemplate();//获取请求
ObjectMapper mapper = new ObjectMapper();//json格式转换
for (int i = 1; i <= 20271; i++) {//爬取页数循环
String forObject = restTemplate.getForObject("https:///?page="+i+"&size=100", String.class);//获取url请求的数据
System.out.println(forObject);
Map map = mapper.readValue(forObject, Map.class);//将Json字符串转为Map对象
Map result = (Map) map.get("result");//获取map中的result
List<Map<String, Object>> data = (List<Map<String, Object>>) result.get("data");//获取result中的data
for (Map<String, Object> datum : data) {//对数据Data进行遍历获取数据()
JSONObject jsonObject = new JSONObject();//创建jSON对象
jsonObject.put("id", datum.get("id"));//存id部分的数据
MultiValueMap<String, String> requestData = new LinkedMultiValueMap<>();
requestData.add("id", datum.get("id").toString());
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(requestData, null);
String s1 = restTemplate.postForObject("https://", request, String.class);根据爬取的数据选择文件爬取方式
Map map1 = mapper.readValue(s1, Map.class);
System.out.println(map1);
Map result1 = (Map) map1.get("result");
List<Map<String, Object>> body = (List<Map<String, Object>>) result1.get("body");
if (body.get(0).get("path")!=null){//对需要下载的地址路径进行拼接
String url = "https://wb.flk.npc.gov.cn/" + body.get(0).get("path");
System.out.println(url);
Object title = result1.get("title");
byte[] file = restTemplate.getForObject(url, byte[].class);//请求下载//路径,并得到字节流对象
String path1 = body.get(0).get("path").toString();//如果提供多种格式的下载,那么取其中一种格式进行下载即可
System.out.println(path1);
String productZipName = path1.substring(path1.lastIndexOf("/"));
String type = productZipName.substring(productZipName.lastIndexOf("."));
File f = new File("E:\\fl\\" + title + type);//将文件按字节流的方式写到//指定存盘中,可以在爬取的数据中获取到文件的title名称作为保存的文件名
FileOutputStream out = new FileOutputStream(f);
out.write(file);
out.close();
}
}
}
}
}