ES7.x使用深分页和浅分页

跟着爸爸走

已于 2023-08-24 17:14:37 修改

阅读量2.2k

点赞数

分类专栏：框架文章标签： ES

于 2019-12-30 10:31:14 首次发布

本文链接：https://blog.csdn.net/heqiang1995/article/details/103762227

版权

框架专栏收录该内容

17 篇文章 0 订阅

订阅专栏

//以关键字开头(优先级最高)
FunctionScoreQueryBuilder.FilterFunctionBuilder[] filterFunctionBuilders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
        new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("grade", "高"), ScoreFunctionBuilders.weightFactorFunction(3)),
        new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("grade", "中"), ScoreFunctionBuilders.weightFactorFunction(2)),
        new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("grade", "低"), ScoreFunctionBuilders.weightFactorFunction(1))};

FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(boolQueryBuilder,filterFunctionBuilders);

sourceBuilder.query(functionScoreQueryBuilder)
        .sort("_score", SortOrder.DESC)
        .sort("happen_time", SortOrder.DESC);

public Page<SyslogOriginalSystem> searchSystemDoc(ParamPI paramPI) throws IOException, ParseException {

        Page page = new Page(paramPI.getPageNum(),paramPI.getPageSize());
        final Scroll scroll = new Scroll(TimeValue.timeValueSeconds(60));

        // 1、创建search请求
        SearchRequest searchRequest = new SearchRequest("syslog_system");
        searchRequest.scroll(scroll);
        // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().trackTotalHits(true);
        //Settings settings = Settings.builder().put("syslog_system.max_result_window", 500000000).build();
        //match分析器，模糊匹配  term精准  multi_match多个字段同时进行匹配
        // bool包含四种操作符，分别是must,should,must_not,query filter查询查询对结果进行缓存
        //sourceBuilder.query(QueryBuilders.termQuery("age", 24));
        //分页设置  浅分页  //深分页 scroll
//        int from  = (paramPI.getPageNum()-1)*paramPI.getPageSize();
//        sourceBuilder.from(from);
        //sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        sourceBuilder.size(paramPI.getPageSize());
        //搜索条件
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        if(paramPI.getKeyword() != null){
            QueryBuilder queryBuilder =  QueryBuilders.matchPhraseQuery("name", paramPI.getKeyword());
            boolQueryBuilder.should(queryBuilder);
        }
        if(paramPI.getAssetId() != null){
            QueryBuilder queryBuilder = QueryBuilders.termQuery("asset_id",paramPI.getAssetId());
            boolQueryBuilder.must(queryBuilder);
        }
        if(paramPI.getLogtype() != null ){
            //elasticsearch 里默认的IK分词器是会将每一个中文都进行了分词的切割，所以你直接想查一整个词  加上.keyword
            QueryBuilder queryBuilder = QueryBuilders.termQuery("type",paramPI.getLogtype());
            boolQueryBuilder.must(queryBuilder);
        }
        QueryBuilder queryBuilder1  = QueryBuilders.rangeQuery("hapeen_time_stamp").from(paramPI.getCustomStartTime().getTime()).to(paramPI.getCustomEndTime().getTime());
        boolQueryBuilder.must(queryBuilder1);

        QueryBuilder queryBuilder2  = QueryBuilders.rangeQuery("s_ip_num").from(paramPI.getSip1()).to(paramPI.getSip2());
        boolQueryBuilder.must(queryBuilder2);

        QueryBuilder queryBuilder3  = QueryBuilders.rangeQuery("d_ip_num").from(paramPI.getDip1()).to(paramPI.getDip2());
        boolQueryBuilder.must(queryBuilder3);
        if(paramPI.getNetStruct() != null && !paramPI.getNetStruct().isEmpty()){
            List<NetStruct> list = paramPI.getNetStruct();
            for(NetStruct netStruct : list){
                QueryBuilder queryBuilder  = QueryBuilders.rangeQuery("d_ip_num").from(netStruct.getStartIp()).to(netStruct.getEndIp());
                boolQueryBuilder.must(queryBuilder);
            }
        }
        sourceBuilder.query(boolQueryBuilder);
        //将请求体加入到请求中
        searchRequest.source(sourceBuilder);

        //3、发送请求
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest,RequestOptions.DEFAULT);
        //处理搜索命中文档结果
        SearchHits hits = searchResponse.getHits();
        long totalHits = hits.getTotalHits().value;

        List<SyslogOriginalSystem> list = new ArrayList<>();
        String scrollId = null;
        int pageNum = paramPI.getPageNum();
        int i = 1;
        while (searchResponse.getHits().getHits().length != 0){
            if(i == pageNum){
                //业务
                for (SearchHit hit : hits.getHits()) {
                    Map<String, Object> sourceAsMap = hit.getSourceAsMap(); // 取成map对象
                    setDataSystem(sourceAsMap,list);
                }
                System.out.println("ES分页查询成功");
                break;
            }

            i++;
            //每次循环完后取得scrollId,用于记录下次将从这个游标开始取数
            scrollId = searchResponse.getScrollId();
            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
            scrollRequest.scroll(scroll);
            searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
        }

        if(scrollId != null){
            //清除滚屏
            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
            //也可以选择setScrollIds()将多个scrollId一起使用
            clearScrollRequest.addScrollId(scrollId);
            ClearScrollResponse  clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
        }

        page.setTotal(totalHits);
        return (Page<SyslogOriginalSystem>) page.setRecords(list);
    }

java连接ES7.x

@Bean
public RestHighLevelClient restHighLevelClient(){
    final CredentialsProvider credentialsProvider =
            new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY,
            new UsernamePasswordCredentials("xxx", "xxx"));

    //解析hostlist配置信息
    String[] split = hostlist.split(",");
    //创建HttpHost数组，其中存放es主机和端口的配置信息
    HttpHost[] httpHostArray = new HttpHost[split.length];
    for(int i=0;i<split.length;i++){
        String item = split[i];
        httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
    }
    RestClientBuilder builder = RestClient.builder(httpHostArray)
            .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                @Override
                public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                }
            });
    //创建RestHighLevelClient客户端
    RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder);
    return restHighLevelClient;
}

所以将字段设置成keyword的时候查询的时候已有的值不会被分词。

说明text类型的字段会被分词，查询的时候如果用拆开查可以查询的到，但是要是直接全部查，就是查询不到。

1、query string search ：查询字符串搜索
2、query DSL ：特定领域的语言
3、query filter : 过滤查询
4、full-text search :（全文检索）
5、phrase search ：（短语搜索）
6、highlight search ：（高亮搜索）
ElasticSearch的几种检索方式_elasticsearch 检索方式_小夙o的博客-CSDN博客

ES使用倒排索引所以比较快
text字段会被分词用于模糊
keyword字段不会被分词用于精确
1, 2”会被拆分成[1, 2]，但是"1,2"是不拆分的，因为之间少了个空格
//搜索条件考虑查询text keyword不同
match分词模糊匹配 match_phrase会被分词
term精准不分词 text字段 terms，in检索
range 范围

扫描所有倒排索引, 性能较差
prefix 前缀检索
wildcard 通配符检索
regexp 正则
fuzzy 纠错检索

boost评分权重令满足某个条件的文档的得分更高, 从而使得其排名更靠前

multi_match多个字段同时进行匹配
bool包含四种操作符，分别是must,should,must_not,query filter过滤查询查询对结果进行缓存
phrase search（短语搜索） full-text search（全文检索）
elasticsearch 里默认的IK分词器是会将每一个中文都进行了分词的切割，所以你直接想查一整个词加上.keyword
ES的默认分词设置是standard 其中f‘-’不在了 simple 分析器当它遇到只要不是字⺟的字符分词
* matchAllQuery:    查询索引库中所有的数据;
* termQuery:       词条查询,词条的完全匹配;
* wildcardQuery:   模糊查询(* ? 通配符);
* queryStringQuery:   将查询条件分词后根据词条匹配    默认返回的并集;
* boolQuery:       must(and), should(or), must_not(not);

sort自定义排序