java爬取数据时候动态获取分页数据

当我们在java代码里请求分页网址的时候需要对分页数据动态获取，写固定数值的话不能灵活，
所以需要先获取总数后在去进行分页

请求工具类，也可以用hutool的

import com.alibaba.fastjson.JSONObject;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.web.client.ResponseErrorHandler;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;

public class ClientHttpUtil {

	/**
	 * 获取连接
	 *
	 * @return
	 */
	public static RestTemplate getRest() {
		RestTemplate rest = new RestTemplate();
		rest.setErrorHandler(new ResponseErrorHandler() {
			@Override
			public boolean hasError(ClientHttpResponse clientHttpResponse) throws IOException {
				return true;
			}

			@Override
			public void handleError(ClientHttpResponse clientHttpResponse) throws IOException {
			}
		});
		return rest;
	}

	/**
	 * 请求网址
	 * 请求网址用
	 * @param url     网址
	 * @param request 请求方式
	 * @param headers 头部数据
	 * @return
	 */
	public static ResponseEntity<String> getresponseEntity(RestTemplate rest, String url, HttpMethod request, HttpHeaders headers) {
		return rest.exchange(url, request, new HttpEntity<String>(headers), String.class);
	}

	/**
	 * 动态分页用
	 * @param rest     连接对象
	 * @param url      网址
	 * @param pageNo   当前页
	 * @param pageSize 显示数量
	 * @param request  请求方式
	 * @param headers  头部信息
	 * @return
	 */
	public static ResponseEntity<String> getresponseEntityByReplacePage(RestTemplate rest, String url, Integer pageNo, Integer pageSize, HttpMethod request, HttpHeaders headers) {
		if (url.contains("pageNo=") || url.contains("pageSize=")) {
			url = url.replace("pageNo=", "pageNo=" + pageNo);
			url = url.replace("pageSize=", "pageSize=" + pageSize);
		}
		return rest.exchange(url, request, new HttpEntity<String>(headers), String.class);
	}

	/**
	 * 获取page页数
	 * @param responseEntity 请求网址后的数据对象
	 * @param size           显示页数
	 * @return
	 */
	public static Integer getPage(ResponseEntity<String> responseEntity, Integer size) {
		if (responseEntity.getStatusCodeValue() == 200) {
			String detail = responseEntity.getBody();
			JSONObject detailObject = JSONObject.parseObject(detail);
			int total = Integer.parseInt(detailObject.get("total").toString());
			return total % size == 0 ? total / size : total / size + 1;
		}
		return null;
	}
}

测试类

package org.springblade.modules.api.controller;

import org.apache.http.client.utils.HttpClientUtils;
import org.springblade.modules.api.utils.ClientHttpUtil;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;

/**
 * @Date 2022/5/13 8:56
 * @remark  动态分页的测试
 */
public class Test {
	public static void main(String[] args) {
		String url = "http://localhost:8080/pageTest?pageNo=&pageSize=";
		// 获取连接对象
		RestTemplate rest = ClientHttpUtil.getRest();
		// 设值请求头部
		HttpHeaders headers = new HttpHeaders();
		// 请求类型
		headers.add("content-type", "application/x-www-form-urlencoded;charset=utf-8");
		// 账号的token数据，没有headers可以为null
		headers.add("token", "");
		// 每页显示的页数，自定义
		Integer size = 3000;
		// getPage获取页数的方法 , getresponseEntityByReplacePage 请求网址的方法动态分页用
		Integer page = ClientHttpUtil.getPage(ClientHttpUtil.getresponseEntityByReplacePage(rest, url, 1, 5, HttpMethod.GET, headers), size);
		// 获取页数后遍历取得所有值
		for (Integer i = 1; i < page + 1; i++) {
			ResponseEntity<String> entity = ClientHttpUtil.getresponseEntityByReplacePage(rest, url, i, size, HttpMethod.GET, headers);
			// 之后解析该数据就可以啦
		}
	}
	/**
	 * 获取请求后返回的对象
	 *
	 * @param responseEntity
	 * @return
	 */
	public static JSONObject getJSONObjectByResponseEntity(ResponseEntity<String> responseEntity) {
		if (responseEntity.getStatusCodeValue() == 200) {
			String body = responseEntity.getBody();
			JSONObject object = JSONObject.parseObject(body);
			if (object.getInteger("code") == 200) {
				return object;
			}
		}
		return null;
	}
}