ES查询量超过一万时候，采用滑动查询来解决

最新推荐文章于 2024-08-26 16:09:06 发布

威威财神

最新推荐文章于 2024-08-26 16:09:06 发布

阅读量581

点赞数 1

文章标签： elasticsearch java

本文链接：https://blog.csdn.net/m0_60524086/article/details/134400894

版权

机缘

项目中ES数据量比较大，这一次有一个bug就是数据超过1W的没有获取到。其实自己知道ES查询返回条目最多10000条，可自己总觉得这种情况很少，所以就没特别注意，这次出现了，就做个笔记，分享给大家，也方便自己下次使用

实际代码

/**

- 查询ES通用方法
  *

- @param userExportHitList

- @param index

- @param boolQueryBuilder

- @Author 张威威

- @Date 2023/10/17
  */
  private void queryEs(List<UserExportHit> userExportHitList, String index, BoolQueryBuilder boolQueryBuilder, List<DataSecuritySpaceHit> dataSecuritySpaceHitList) throws IOException {
  //处理数据量大于10000的时候,采用滚动查询
  Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
  SearchRequest searchRequest = new SearchRequest(index);
  SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
  //sourceBuilder.fetchSource(index,null);
  sourceBuilder.query(boolQueryBuilder);
  sourceBuilder.size(10000);
  IndicesOptions indicesOptions = IndicesOptions.LENIENT_EXPAND_OPEN;
  searchRequest.indicesOptions(indicesOptions);
  //加入scroll和构造器
  searchRequest.scroll(scroll);
  searchRequest.source(sourceBuilder);
  log.info("~~~~~~~查询ES请求体 query:{}~~~~~~~~~", searchRequest);
  List<String> scrollIdList = new ArrayList<>();
  RestHighLevelClient client = (RestHighLevelClient) SpringUtils.getBean(ElasticsearchAuditConfig.NAME);
  SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
  //拿到都一个scrollId（游标）
  String scrollId = searchResponse.getScrollId();
  SearchHit[] hits = searchResponse.getHits().getHits();
  //添加下标进集合
  scrollIdList.add(scrollId);

  try {
      while (ArrayUtils.isNotEmpty(hits)) {
          for (SearchHit hit : hits) {
              Map<String, Object> map = hit.getSourceAsMap();
              Long systemTime = (Long) map.get("system_time");
              String oprUserAccount = (String) map.get("opr_user_account");
              if (null != systemTime && StringUtils.isNotBlank(oprUserAccount)) {
                  Date date = new Date(systemTime);
                  UserExportHit userExportHit = new UserExportHit();
                  userExportHit.setExportTime(date);
                  userExportHit.setOprUserAccount(oprUserAccount);

  ```
              String oprMenuname = (String) map.get("opr_menu_name");
              String systemIp = (String) map.get("system_ip");
              String oprUserName = (String) map.get("opr_user_name");
              String oprOrgCode = (String) map.get("opr_org_code");
              String id = hit.getId();
              Integer exportnum = (Integer) map.get("export_num");
              userExportHit.setSystemTime(systemTime);
              userExportHit.setOprMenuName(oprMenuname);
              userExportHit.setSystemIp(systemIp);
              userExportHit.setOprUserName(oprUserName);
              userExportHit.setOprOrgCode(oprOrgCode);
              userExportHit.setExportNum(exportnum);
              userExportHit.setLogId(id);
              userExportHitList.add(userExportHit);
          }
  
          String account = (String) map.get("account");
          String opt = (String) map.get("opt");
          String targetIp = (String) map.get("targetIp");
          Long eventTimestamp = (Long) map.get("eventTimestamp");
          if (StringUtils.isNotBlank(account) && StringUtils.isNotBlank(opt) && null != eventTimestamp) {
              DataSecuritySpaceHit dataSecuritySpaceHit = new DataSecuritySpaceHit();
              dataSecuritySpaceHit.setAccount(account);
              Date date = new Date(eventTimestamp);
              dataSecuritySpaceHit.setTime(date);
              dataSecuritySpaceHit.setOpt(opt);
              dataSecuritySpaceHit.setTargetIp(targetIp);
              dataSecuritySpaceHitList.add(dataSecuritySpaceHit);
          }
      }
      if (hits.length < 10000) {
          log.info("es滚动完了，结束就行");
          break;
      }
      //继续滚动
      SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId);
      searchScrollRequest.scroll(scroll);
      SearchResponse response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT);
      scrollId = response.getScrollId();
      hits = response.getHits().getHits();
      scrollIdList.add(scrollId);
  }
  ```

  } finally {
      ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
      clearScrollRequest.setScrollIds(scrollIdList);
      client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
  }
  }