Elasticsearch Scroll查询

星夜孤帆

于 2023-12-28 16:44:39 发布

阅读量566

点赞数 8

文章标签： elasticsearch 大数据搜索引擎

本文链接：https://blog.csdn.net/qq_38826019/article/details/135273294

版权

    @Override
    public void scrollSearch(String index) {
        // 创建 JestClient 客户端实例
        try {
            // 构建初始的搜索请求
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(QueryBuilders.matchAllQuery());
            searchSourceBuilder.size(10);
            Search search = new Search.Builder(searchSourceBuilder.toString())
                    .addIndex(index)
                    .setParameter(Parameters.SCROLL, "1m")
                    .build();

            // 执行初始搜索请求
            SearchResult searchResult = client.execute(search);
            String scrollId = searchResult.getJsonObject().get("_scroll_id").getAsString();

            // 处理初始搜索结果
            JestResult result = searchResult;
            while (result.isSucceeded()) {
                // 从 JestResult 中获取滚动查询结果
                JsonElement hitsElement = result.getJsonObject().get("hits").getAsJsonObject();
                JsonArray hitsArray = hitsElement.getAsJsonObject().get("hits").getAsJsonArray();

                for (JsonElement hitElement : hitsArray) {
                    JsonObject hitObject = hitElement.getAsJsonObject();
                    String id = hitObject.get("_id").getAsString();
                    JsonObject sourceObject = hitObject.get("_source").getAsJsonObject();
                    // 处理每个 hit
                    log.info("id: {}", id);
                    log.info("source: {}", sourceObject.toString());

                    // 进行其他处理操作
                    // ...
                }

                // 执行滚动请求
                SearchScroll scroll = new SearchScroll.Builder(scrollId, "1m").build();
                result = client.execute(scroll);
                log.info("client execute");

                scrollId = result.getJsonObject().get("_scroll_id").getAsString();
            }
            log.info("清除滚动上下文 开始");
            // 清除滚动上下文
            ClearScroll clearScroll = new ClearScroll.Builder().addScrollId(scrollId).build();
            client.execute(clearScroll);
            log.info("清除滚动上下文 结束");
        } catch (Exception e) {
            log.error("scrollSearch catch error index: {}", index, e);
        }
    }

public void scrollSearch(String index) {
        try {
            int scrollSize = 10;
            Long scrollTime = 1L;

            // 创建初始的搜索请求
            SearchRequest searchRequest = new SearchRequest(index);
            searchRequest.scroll(TimeValue.timeValueMinutes(scrollTime));

            /// 设置搜索条件
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(QueryBuilders.matchAllQuery());
            searchSourceBuilder.size(scrollSize);
            searchRequest.source(searchSourceBuilder);

            // 执行初始搜索请求
            RestHighLevelClient client = getRestHighLevelClient();
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            String scrollId = searchResponse.getScrollId();
            List<SearchHit> hits = Arrays.asList(searchResponse.getHits().getHits());
            while (CollectionUtils.isNotEmpty(hits)) {
                for (SearchHit hit : hits) {
                    String indexName = hit.getIndex();
                    String id = hit.getId();
                    Map<String, Object> source = hit.getSourceAsMap();
                    log.info("scrollSearch indexName: {}, id: {}, source: {}", id, indexName, id, JSON.toJSONString(source));
                }

                // 执行scroll请求继续滚动查询下一批数据
                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                scrollRequest.scroll(TimeValue.timeValueMinutes(scrollTime));
                searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
                scrollId = searchResponse.getScrollId();
                hits = Arrays.asList(searchResponse.getHits().getHits());
                log.info("scrollSearch client execute");
            }

            // 清除scroll上下文
            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
            clearScrollRequest.addScrollId(scrollId);
            ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
            boolean succeeded = clearScrollResponse.isSucceeded();
            log.info("scrollSearch clearScroll succeeded: {}", succeeded);
        } catch (Exception e) {
            log.error("scrollSearch catch error index: {}", index, e);
        }
    }

参考博客、视频教程、视频教程