使用elasticsearch 2.4 版本,想分页遍历所有的文档,因为 from + size 有范围限制,且效率低下,因此采用 scroll,代码如下,结果每次打印出来的数据一模一样,但是scroll_id 的值是最新的
public static void main(String[] args) {
try {
String url = "http://ip:port/index/type/_search";
String oldEsResult = sendGet(url, "scroll=20s&search_type=scan&size=10");
ObjectMapper mapper1 = new ObjectMapper();
String scroll_id;
int totalCount = NumberUtils.toInt(mapper1.readTree(oldEsResult).findValue("hits").findValue("total").toString());
scroll_id = mapper1.readTree(oldEsResult).findValue("_scroll_id").toString();
System.out.println("totalCount is-------" + totalCount);
int size = 100;
int numbers = totalCount / size; //页数
String params = "scroll=20s&scroll_id=";
for(int i = 0; i < numbers; i++){
String param = params + scroll_id;
oldEsResult = sendGet(url, param);
if(StringUtils.isBlank(oldEsResult)){
continue;
}
ObjectMapper mapper2 = new ObjectMapper();
scroll_id = mapper2.readTree(oldEsResult).findValue("_scroll_id").toString();
JsonNode node = mapper2.readTree(oldEsResult).findValue("hits").findValue("hits");
for (int j = 0; j < node.size(); j++) {
System.out.println(node.get(j).get("_source"));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static String sendGet(String url, String param) {
String result = "";
BufferedReader in = null;
try {
String urlNameString = url + "?" + param;
URL realUrl = new URL(urlNameString);
// 打开和URL之间的连接
URLConnection connection = realUrl.openConnection();
// 设置通用的请求属性
connection.setRequestProperty("accept", "*/*");
connection.setRequestProperty("connection", "Keep-Alive");
connection.setRequestProperty("user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
// 建立实际的连接
connection.connect();
// 获取所有响应头字段
Map<String, List<String>> map = connection.getHeaderFields();
// 定义 BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
System.out.println("发送GET请求出现异常!" + e);
e.printStackTrace();
}
// 使用finally块来关闭输入流
finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e2) {
e2.printStackTrace();
}
}
return result;
}