Elasticsearch分页代码(基于scroll api全量遍历elasticsearch数据)

全量遍历elasticsearch6.3索引数据的工具类，可以直接使用EsScrollQueryUtil和HttpUtil完成。

 

https://github.com/memoryFuhao/elasticsearch_client  (打个广告 以上链接是本人开发的一个es客户端工具，支持es大部分 CRUD操作  分页、分组、嵌套分组、and or ···，有需要的朋友可以pull代码直接使用)

 

直接上代码：

 

查询类EsScrollQueryUtil：

/**
 * Created by memory_fu on 2019/07/25.
 */
public class EsScrollQueryUtil {

	private static final String defaultScrollTime = "5m";
	private static final String defaultPort = "9200";
	private String scrollId = new String();

	/**
	 * 全量遍历es数据
	 * 
	 * @param indexName
	 *            索引名称
	 * @param ip
	 *            安装es地址
	 * @param size
	 *            每次查询es数据条数
	 * @return
	 * @throws IOException
	 */
	public List<JSONObject> queryData(String indexName, String ip, int size) throws IOException {
		return queryData(indexName, ip, defaultScrollTime, size);
	}

	/**
	 * 全量遍历es数据
	 * 
	 * @param indexName
	 *            索引名称
	 * @param ip
	 *            安装es地址
	 * @param scrollTime
	 *            scrollId保存时间
	 * @param size
	 *            每次查询es数据条数
	 * @return
	 * @throws IOException
	 */
	public List<JSONObject> queryData(String indexName, String ip, String scrollTime, int size) throws IOException {

		List<JSONObject> result = new ArrayList<>();
		if (StringUtils.isEmpty(this.scrollId)) {
			result = getScrollId(indexName, ip, size);
		} else {
			result = getScrollData(indexName, ip);
		}
		return result;
	}

	private List<JSONObject> getScrollId(String indexName, String ip, int size) throws IOException {
		// 获取srcollId和数据
		String url = getUrl(indexName, ip, (String) null);

		JSONObject jsonObject = new JSONObject();
		jsonObject.put("size", size);
		String queryStr = jsonObject.toJSONString();

		// System.out.println(url + " ____ " + queryStr);
		String responseStr = HttpUtil.requestMethod(HttpUtil.HTTP_POST, url, queryStr);

		List<JSONObject> analysisData = analysisData(responseStr);

		return analysisData;
	}

	private List<JSONObject> getScrollData(String indexName, String ip) throws IOException {

		String url = getUrl(indexName, ip, defaultScrollTime);

		JSONObject jsonObject = new JSONObject();
		jsonObject.put("scroll", defaultScrollTime);
		jsonObject.put("scroll_id", this.scrollId);
		String queryStr = jsonObject.toJSONString();

		// System.out.println(url + " ____ " + queryStr);
		String responseStr = HttpUtil.requestMethod(HttpUtil.HTTP_POST, url, queryStr);

		List<JSONObject> analysisData = analysisData(responseStr);

		return analysisData;
	}

	private List<JSONObject> analysisData(String responseStr) {
		List<JSONObject> list = new ArrayList<>();

		JSONObject jsonObject = JSONObject.parseObject(responseStr);
		if (null == jsonObject) {
			return list;
		}

		this.scrollId = jsonObject.getString("_scroll_id");

		JSONObject parentHits = jsonObject.getJSONObject("hits");
		if (null == parentHits) {
			return list;
		}
		JSONArray subHits = parentHits.getJSONArray("hits");
		if (null == subHits) {
			return list;
		}

		for (int i = 0; i < subHits.size(); i++) {
			JSONObject object = subHits.getJSONObject(i);
			String id = object.getString("_id");
			JSONObject source = object.getJSONObject("_source");
			source.put("id", id);
			list.add(source);
		}

		return list;
	}

	private String getUrl(String indexName, String ip, String port, String scrollTime) {

		if (StringUtils.isEmpty(scrollTime)) {
			return new String("http://" + ip + ":" + port + "/" + indexName + "/_search?scroll=" + defaultScrollTime);
		}
		return new String("http://" + ip + ":" + port + "/_search/scroll");
	}

	private String getUrl(String indexName, String ip, String scrollTime) {
		return getUrl(indexName, ip, defaultPort, scrollTime);
	}

	private void clear() {
		this.scrollId = null;
	}

	public static void main(String[] args) throws IOException {
		String[] indexs = new String[] { "vlpr_result", "summary_result", "objext_result", "face_result",
				"bike_result" };

		EsScrollQueryUtil queryUtil = new EsScrollQueryUtil();
		for (String index : indexs) {
			int count = 0;
			while (true) {
				List<JSONObject> list = queryUtil.queryData(index, "172.16.1.68", 1000);
				count += list.size();
				System.out.println(JSONObject.toJSONString(list));

				if (list.size() == 0) {
					System.out.println(index + " data count: " + count);
					queryUtil.clear();// 一个索引数据遍历完成后必须调用此方法清除scrollId
					break;
				}
			}
		}

	}

}

 

Http请求工具类HttpUtil：

/**
 * Created by memory_fu on 2019/6/11.
 */
public class HttpUtil {

    // post请求
    public static final String HTTP_POST = "POST";
    // get请求
    public static final String HTTP_GET = "GET";
    // utf-8字符编码
    public static final String CHARSET_UTF_8 = "UTF-8";
    // HTTP内容类型
    public static final String CONTENT_TYPE_TEXT_HTML = "text/xml";
    // HTTP内容类型
    private static final String CONTENT_TYPE = "application/json";
    // 请求超时时间
    public static final int SEND_REQUEST_TIME_OUT = 20000;
    // 将读超时时间
    public static final int READ_TIME_OUT = 20000;

    /**
     * @param requestType    请求类型
     * @param urlStr        请求地址
     * @param body        请求发送内容
     * @return 返回内容
     */
    public static String requestMethod(String requestType, String urlStr, String body)
        throws IOException {
        boolean isDoInput = false;
        if (body != null && body.length() > 0) {
            isDoInput = true;
        }
        OutputStream outputStream = null;
        OutputStreamWriter outputStreamWriter = null;
        InputStream inputStream = null;
        InputStreamReader inputStreamReader = null;
        BufferedReader reader = null;
        StringBuilder stringBuilder = new StringBuilder();
        String tempLine = null;
        URL url = new URL(urlStr);
        URLConnection urlConnection = url.openConnection();
        HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
        if (isDoInput) {
            httpURLConnection.setDoOutput(true);
            httpURLConnection.setRequestProperty("Content-Length", String.valueOf(body.length()));
        }
        httpURLConnection.setDoInput(true);
        httpURLConnection.setConnectTimeout(SEND_REQUEST_TIME_OUT);
        httpURLConnection.setReadTimeout(READ_TIME_OUT);
        httpURLConnection.setUseCaches(false);
        httpURLConnection.setRequestProperty("Accept-Charset", CHARSET_UTF_8);
        httpURLConnection.setRequestProperty("Content-Type", CONTENT_TYPE);
        httpURLConnection.setRequestMethod(requestType);
        httpURLConnection.connect();
        if (isDoInput) {
            outputStream = httpURLConnection.getOutputStream();
            outputStreamWriter = new OutputStreamWriter(outputStream);
            outputStreamWriter.write(body);
            outputStreamWriter.flush();// 刷新
        }
        if (httpURLConnection.getResponseCode() >= 300) {
            
        }
        if (httpURLConnection.getResponseCode() == HttpURLConnection.HTTP_OK) {
            inputStream = httpURLConnection.getInputStream();
            inputStreamReader = new InputStreamReader(inputStream, CHARSET_UTF_8);
            reader = new BufferedReader(inputStreamReader);

            while ((tempLine = reader.readLine()) != null) {
                stringBuilder.append(tempLine);
            }
        }
        if (outputStreamWriter != null) {
            outputStreamWriter.close();
        }
        if (outputStream != null) {
            outputStream.close();
        }
        if (reader != null) {
            reader.close();
        }
        if (inputStreamReader != null) {
            inputStreamReader.close();
        }
        if (inputStream != null) {
            inputStream.close();
        }
        return stringBuilder.toString();
    }

    /**
     * 请求重试机制 二次请求需等待0.5秒
     *
     * @param requestType 请求类型
     * @param action 请求接口
     * @param params 请求参数 json格式
     * @param retry true-重试, false-不重试
     */
    public static String retryHandler(String requestType, String action, String params,
        boolean retry) {
        try {
            return requestMethod(requestType, action, params);
        } catch (Exception e) {
            if (retry) {
                try {
                    Thread.sleep(500);
                    return requestMethod(requestType, action, params);
                } catch (Exception e1) {
                	e1.printStackTrace();
                }
            }
        }
        return "-100001"; //服务端接口出错
    }
}