全量遍历elasticsearch6.3索引数据的工具类,可以直接使用EsScrollQueryUtil和HttpUtil完成。
https://github.com/memoryFuhao/elasticsearch_client (打个广告 以上链接是本人开发的一个es客户端工具,支持es大部分 CRUD操作 分页、分组、嵌套分组、and or ···,有需要的朋友可以pull代码直接使用)
直接上代码:
查询类EsScrollQueryUtil:
/**
* Created by memory_fu on 2019/07/25.
*/
public class EsScrollQueryUtil {
private static final String defaultScrollTime = "5m";
private static final String defaultPort = "9200";
private String scrollId = new String();
/**
* 全量遍历es数据
*
* @param indexName
* 索引名称
* @param ip
* 安装es地址
* @param size
* 每次查询es数据条数
* @return
* @throws IOException
*/
public List<JSONObject> queryData(String indexName, String ip, int size) throws IOException {
return queryData(indexName, ip, defaultScrollTime, size);
}
/**
* 全量遍历es数据
*
* @param indexName
* 索引名称
* @param ip
* 安装es地址
* @param scrollTime
* scrollId保存时间
* @param size
* 每次查询es数据条数
* @return
* @throws IOException
*/
public List<JSONObject> queryData(String indexName, String ip, String scrollTime, int size) throws IOException {
List<JSONObject> result = new ArrayList<>();
if (StringUtils.isEmpty(this.scrollId)) {
result = getScrollId(indexName, ip, size);
} else {
result = getScrollData(indexName, ip);
}
return result;
}
private List<JSONObject> getScrollId(String indexName, String ip, int size) throws IOException {
// 获取srcollId和数据
String url = getUrl(indexName, ip, (String) null);
JSONObject jsonObject = new JSONObject();
jsonObject.put("size", size);
String queryStr = jsonObject.toJSONString();
// System.out.println(url + " ____ " + queryStr);
String responseStr = HttpUtil.requestMethod(HttpUtil.HTTP_POST, url, queryStr);
List<JSONObject> analysisData = analysisData(responseStr);
return analysisData;
}
private List<JSONObject> getScrollData(String indexName, String ip) throws IOException {
String url = getUrl(indexName, ip, defaultScrollTime);
JSONObject jsonObject = new JSONObject();
jsonObject.put("scroll", defaultScrollTime);
jsonObject.put("scroll_id", this.scrollId);
String queryStr = jsonObject.toJSONString();
// System.out.println(url + " ____ " + queryStr);
String responseStr = HttpUtil.requestMethod(HttpUtil.HTTP_POST, url, queryStr);
List<JSONObject> analysisData = analysisData(responseStr);
return analysisData;
}
private List<JSONObject> analysisData(String responseStr) {
List<JSONObject> list = new ArrayList<>();
JSONObject jsonObject = JSONObject.parseObject(responseStr);
if (null == jsonObject) {
return list;
}
this.scrollId = jsonObject.getString("_scroll_id");
JSONObject parentHits = jsonObject.getJSONObject("hits");
if (null == parentHits) {
return list;
}
JSONArray subHits = parentHits.getJSONArray("hits");
if (null == subHits) {
return list;
}
for (int i = 0; i < subHits.size(); i++) {
JSONObject object = subHits.getJSONObject(i);
String id = object.getString("_id");
JSONObject source = object.getJSONObject("_source");
source.put("id", id);
list.add(source);
}
return list;
}
private String getUrl(String indexName, String ip, String port, String scrollTime) {
if (StringUtils.isEmpty(scrollTime)) {
return new String("http://" + ip + ":" + port + "/" + indexName + "/_search?scroll=" + defaultScrollTime);
}
return new String("http://" + ip + ":" + port + "/_search/scroll");
}
private String getUrl(String indexName, String ip, String scrollTime) {
return getUrl(indexName, ip, defaultPort, scrollTime);
}
private void clear() {
this.scrollId = null;
}
public static void main(String[] args) throws IOException {
String[] indexs = new String[] { "vlpr_result", "summary_result", "objext_result", "face_result",
"bike_result" };
EsScrollQueryUtil queryUtil = new EsScrollQueryUtil();
for (String index : indexs) {
int count = 0;
while (true) {
List<JSONObject> list = queryUtil.queryData(index, "172.16.1.68", 1000);
count += list.size();
System.out.println(JSONObject.toJSONString(list));
if (list.size() == 0) {
System.out.println(index + " data count: " + count);
queryUtil.clear();// 一个索引数据遍历完成后必须调用此方法清除scrollId
break;
}
}
}
}
}
Http请求工具类HttpUtil:
/**
* Created by memory_fu on 2019/6/11.
*/
public class HttpUtil {
// post请求
public static final String HTTP_POST = "POST";
// get请求
public static final String HTTP_GET = "GET";
// utf-8字符编码
public static final String CHARSET_UTF_8 = "UTF-8";
// HTTP内容类型
public static final String CONTENT_TYPE_TEXT_HTML = "text/xml";
// HTTP内容类型
private static final String CONTENT_TYPE = "application/json";
// 请求超时时间
public static final int SEND_REQUEST_TIME_OUT = 20000;
// 将读超时时间
public static final int READ_TIME_OUT = 20000;
/**
* @param requestType 请求类型
* @param urlStr 请求地址
* @param body 请求发送内容
* @return 返回内容
*/
public static String requestMethod(String requestType, String urlStr, String body)
throws IOException {
boolean isDoInput = false;
if (body != null && body.length() > 0) {
isDoInput = true;
}
OutputStream outputStream = null;
OutputStreamWriter outputStreamWriter = null;
InputStream inputStream = null;
InputStreamReader inputStreamReader = null;
BufferedReader reader = null;
StringBuilder stringBuilder = new StringBuilder();
String tempLine = null;
URL url = new URL(urlStr);
URLConnection urlConnection = url.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
if (isDoInput) {
httpURLConnection.setDoOutput(true);
httpURLConnection.setRequestProperty("Content-Length", String.valueOf(body.length()));
}
httpURLConnection.setDoInput(true);
httpURLConnection.setConnectTimeout(SEND_REQUEST_TIME_OUT);
httpURLConnection.setReadTimeout(READ_TIME_OUT);
httpURLConnection.setUseCaches(false);
httpURLConnection.setRequestProperty("Accept-Charset", CHARSET_UTF_8);
httpURLConnection.setRequestProperty("Content-Type", CONTENT_TYPE);
httpURLConnection.setRequestMethod(requestType);
httpURLConnection.connect();
if (isDoInput) {
outputStream = httpURLConnection.getOutputStream();
outputStreamWriter = new OutputStreamWriter(outputStream);
outputStreamWriter.write(body);
outputStreamWriter.flush();// 刷新
}
if (httpURLConnection.getResponseCode() >= 300) {
}
if (httpURLConnection.getResponseCode() == HttpURLConnection.HTTP_OK) {
inputStream = httpURLConnection.getInputStream();
inputStreamReader = new InputStreamReader(inputStream, CHARSET_UTF_8);
reader = new BufferedReader(inputStreamReader);
while ((tempLine = reader.readLine()) != null) {
stringBuilder.append(tempLine);
}
}
if (outputStreamWriter != null) {
outputStreamWriter.close();
}
if (outputStream != null) {
outputStream.close();
}
if (reader != null) {
reader.close();
}
if (inputStreamReader != null) {
inputStreamReader.close();
}
if (inputStream != null) {
inputStream.close();
}
return stringBuilder.toString();
}
/**
* 请求重试机制 二次请求需等待0.5秒
*
* @param requestType 请求类型
* @param action 请求接口
* @param params 请求参数 json格式
* @param retry true-重试, false-不重试
*/
public static String retryHandler(String requestType, String action, String params,
boolean retry) {
try {
return requestMethod(requestType, action, params);
} catch (Exception e) {
if (retry) {
try {
Thread.sleep(500);
return requestMethod(requestType, action, params);
} catch (Exception e1) {
e1.printStackTrace();
}
}
}
return "-100001"; //服务端接口出错
}
}