ES Scroll 遍历查询

业务场景中遇到需要取一段时间范围内的数据，一开始选择深度检索，每次查询10000条，发现查询越来越慢，显然From+ size 不适用于此场景这是需要用到search_after或者Scroll

因为需要取一段时间内的全量或数据量很大时遍历结果数据所以选择了Scroll

1. 创建ES连接

import lombok.Data;

import org.apache.http.HttpHost;

import org.apache.http.client.config.RequestConfig;

import org.elasticsearch.client.RestClient;

import org.elasticsearch.client.RestClientBuilder;

import org.elasticsearch.client.RestHighLevelClient;

import org.springframework.boot.context.properties.ConfigurationProperties;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Configuration;

import java.net.InetAddress;

import java.net.UnknownHostException;

@Configuration

@ConfigurationProperties(prefix = "test.config.es")

@Data

public class ElasticlLikwsConfig {

private String ip;

private String port;

private String indexName = "cdr";

private int timeOut = 5 * 60 * 1000;

private int scrollTime = 180000;

@Bean

public RestHighLevelClient restHighLevelClient() throws UnknownHostException {

String[] ips = ip.split(",");

HttpHost[] hosts = new HttpHost[ips.length];

for (int i = 0; i < ips.length; i++) {

hosts[i] = new HttpHost(InetAddress.getByName(ips[i]), Integer.parseInt(port));

}

RestClientBuilder.RequestConfigCallback requestConfigCallback = new RestClientBuilder.RequestConfigCallback() {

@Override

public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder builder) {

return builder.setConnectTimeout(timeOut).setSocketTimeout(timeOut).setConnectionRequestTimeout(timeOut);

}

};

RestClient restTargetClient = RestClient.builder(hosts).setMaxRetryTimeoutMillis(timeOut).setRequestConfigCallback(requestConfigCallback).build();

RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restTargetClient);

return restHighLevelClient;

}

}

2. 应用

@Autowired

private RestHighLevelClient restHighLevelClient;

@Value("${es.pageSize:10000}")

private Integer pageSize;

public void esSearch() {

//转换json

BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();

SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

searchSourceBuilder.size(pageSize);

FetchSourceContext fetchSourceContext = new FetchSourceContext(true, EsQueryConsts.getLikwsCloums(), Strings.EMPTY_ARRAY);

searchSourceBuilder.fetchSource(fetchSourceContext);

Date beginDate = new Date();

Date endDate = new Date();

//时间

boolQueryBuilder.must(QueryBuilders.rangeQuery("begin_time").gte(beginDate.getTime()).lte(endDate.getTime()));

//语种

List<String> langs = searchParameter.getBaseParamter().getLangs();

BoolQueryBuilder builder = new BoolQueryBuilder();

for (String lang : langs) {

builder.should(QueryBuilders.termQuery(EsQueryConsts.LANG, lang));

}

boolQueryBuilder.must(builder);

//设置返回条数和排序方式

searchSourceBuilder.sort(new FieldSortBuilder("begin_time").order(SortOrder.DESC));

//索引信息

SearchRequest request = new SearchRequest(elasticlLikwsConfig.getIndexName() + "/" + TYPE);

searchSourceBuilder.timeout(new TimeValue(elasticlLikwsConfig.getTimeOut(), TimeUnit.SECONDS));

//滚动查询的Scroll

Scroll scroll = new Scroll(TimeValue.timeValueMillis(elasticlLikwsConfig.getScrollTime()));

request.source(searchSourceBuilder.query(boolQueryBuilder));

request.scroll(scroll);

logger.debug("ES查询条件：" + searchSourceBuilder.toString());

//存储scroll的list

List<String> scrollIdList = new ArrayList<>();

SearchResponse searchResponse = null;

try {

searchResponse = restLikwsHighLevelClient.search(request);

if (searchResponse.status() == RestStatus.OK) {

//第一个ScrollId（游标）

String scrollId = searchResponse.getScrollId();

//hits结果

SearchHits hits = searchResponse.getHits();

if (hits.getHits() == null || hits.getHits().length == 0) {

//TODO 业务处理

} else {

//TODO 业务处理

}

scrollIdList.add(scrollId);

try {

//滚动查询将SearchHit封装到result中

while (ArrayUtils.isNotEmpty(hits.getHits())) {

//说明滚动完了，返回结果即可

if (hits.getHits().length < pageSize) {

break;

}

//继续滚动，根据上一个游标，得到这次开始查询位置

SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId);

searchScrollRequest.scroll(scroll);

//结果

SearchResponse searchScrollResponse = restHighLevelClient.searchScroll(searchScrollRequest);

//定位游标

scrollId = searchScrollResponse.getScrollId();

hits = searchScrollResponse.getHits();

//TODO 业务处理

scrollIdList.add(scrollId);

}

} finally {

//清理scroll,释放资源

ClearScrollRequest clearScrollRequest = new ClearScrollRequest();

clearScrollRequest.setScrollIds(scrollIdList);

restLikwsHighLevelClient.clearScroll(clearScrollRequest);

}

}

} catch (Exception e) {

logger.error("es查询出错：{}", e);

}