Java调用ElasticSearch 7.2.1 保存、统计、多字段分组聚合

草宝虫

已于 2022-09-06 13:48:14 修改

阅读量8.9k

点赞数 7

分类专栏： java elk # Elasticsearch 文章标签： ElasticSearch 多字段分组聚合

于 2019-08-06 14:11:18 首次发布

本文链接：https://blog.csdn.net/whq12789/article/details/98612670

版权

java 同时被 3 个专栏收录

61 篇文章 0 订阅

订阅专栏

elk

6 篇文章 1 订阅

订阅专栏

Elasticsearch

4 篇文章 0 订阅

订阅专栏

maven引用

注意版本与es版本一致

<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.2.1</version>
    <exclusions>
        <exclusion>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
        </exclusion>
    </exclusions>
</dependency>


<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>7.2.1</version>
</dependency>

配置

spring:
  data:
    elasticsearch:
      ip: 10.0.197.198
      port: 9200
      scheme: http

配置类

import lombok.Data;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;

@Configuration
@ConfigurationProperties(prefix = "spring.data.elasticsearch")
@Data
public class ElasticsearchConfig {

    private String ip;
    private Integer port;
    private String scheme;
    public RestHighLevelClient getClient() {
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(ip, port, scheme)));

        return client;
    }
}

保存数据方法

public Boolean setPageAccess(Map map) {
    try(RestHighLevelClient client= elasticsearchConfig.getClient()) {
        // 1、创建索引请求
        IndexRequest indexRequest = new IndexRequest(
                "test");     //文档id
        indexRequest.id(StringUtil.getUUID());
        map.put("time",new Date());

        // 2、准备文档数据
        // 方式一：直接给JSON串
        String jsonString = JSON.toJSONString(map);
        indexRequest.source(jsonString, XContentType.JSON);
        //4、发送请求
        IndexResponse indexResponse = null;
        try {
            // 同步方式
            indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
        } catch (ElasticsearchException e) {
            // 捕获，并处理异常
            //判断是否版本冲突、create但文档已存在冲突
            if (e.status() == RestStatus.CONFLICT) {
                log.error("冲突了\n" + e.getDetailedMessage());
            }
            log.error("索引异常", e);
            return false;
        }
    }catch (Exception e){
        e.printStackTrace();
        return false;
    }
    return true;
}

参照官方的引用方式会报错

java.lang.NoClassDefFoundError: org/elasticsearch/common/xcontent/DeprecationHandler

解决办法：排除旧的es引用

<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>7.2.1</version>
    <exclusions>
        <exclusion>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
        </exclusion>
    </exclusions>
</dependency>


<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>7.2.1</version>
</dependency>

分组统计

注意：分组如果在字符串字段上，需要建立字段对应的.keyword字段，该字段支持聚合处理，直接用字符串字段会报错。

try(RestHighLevelClient client= elasticsearchConfig.getClient()) {

    // 1、创建search请求
    //SearchRequest searchRequest = new SearchRequest();
    SearchRequest searchRequest = new SearchRequest(INDEX_PAGEACCESS);

    // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

    //sourceBuilder.query();
    sourceBuilder.size(0);

    //加入聚合
    //字段值项分组聚合
    TermsAggregationBuilder aggregation = AggregationBuilders.terms("by_method")
            .field("method.keyword").order(BucketOrder.aggregation("count", true));
    //计算每组的平均balance指标
    aggregation.subAggregation(AggregationBuilders.count("count")
            .field("_sessionid"));
    sourceBuilder.aggregation(aggregation);

    searchRequest.source(sourceBuilder);

    //3、发送请求
    SearchResponse searchResponse = client.search(searchRequest,RequestOptions.DEFAULT);

    //4、处理响应
    //搜索结果状态信息
    if(RestStatus.OK.equals(searchResponse.status())) {
        // 获取聚合结果
        Aggregations aggregations = searchResponse.getAggregations();
        Terms byAgeAggregation = aggregations.get("by_method");
        log.info("aggregation by_age 结果");
        log.info("docCountError: " + byAgeAggregation.getDocCountError());
        log.info("sumOfOtherDocCounts: " + byAgeAggregation.getSumOfOtherDocCounts());
        log.info("------------------------------------");
        for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
            log.info("key: " + buck.getKeyAsString());
            log.info("docCount: " + buck.getDocCount());
            log.info("docCountError: " + buck.getDocCountError());
            //取子聚合
            ParsedValueCount averageBalance = buck.getAggregations().get("count");

            log.info("average_balance: " + averageBalance.getValue());
            log.info("------------------------------------");
        }
        //直接用key 来去分组
        /*Bucket elasticBucket = byCompanyAggregation.getBucketByKey("24");
        Avg averageAge = elasticBucket.getAggregations().get("average_age");
        double avg = averageAge.getValue();*/

    }
    return new ArrayList();
}catch (Exception e){
    e.printStackTrace();
    return null;
}

查询后再统计

try(RestHighLevelClient client= elasticsearchConfig.getClient()) {

    // 1、创建search请求
    //SearchRequest searchRequest = new SearchRequest();
    SearchRequest searchRequest = new SearchRequest(INDEX_PAGEACCESS);

    // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

    //构造QueryBuilder
    QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("method.keyword", "web_event_srv.upload");
    sourceBuilder.query(matchQueryBuilder);
    sourceBuilder.size(0);

    //加入聚合
    //字段值项分组聚合
    TermsAggregationBuilder aggregation = AggregationBuilders.terms("by_fngroup")
            .field("fngroup.keyword").order(BucketOrder.aggregation("count", true));
    //计算每组的平均balance指标
    aggregation.subAggregation(AggregationBuilders.count("count")
            .field("sessionid"));
    sourceBuilder.aggregation(aggregation);

    searchRequest.source(sourceBuilder);

    //3、发送请求
    SearchResponse searchResponse = client.search(searchRequest,RequestOptions.DEFAULT);

    //4、处理响应
    //搜索结果状态信息
    List<Map> result=new ArrayList<>();
    if(RestStatus.OK.equals(searchResponse.status())) {
        // 获取聚合结果
        Aggregations aggregations = searchResponse.getAggregations();
        Terms byAgeAggregation = aggregations.get("by_fngroup");
        for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
            Map map=new HashMap();
            map.put("name", buck.getKeyAsString());
            //取子聚合
            ParsedValueCount averageBalance = buck.getAggregations().get("count");

            map.put("count",averageBalance.getValue());
            result.add(map);
        }
    }
    return result;
}catch (Exception e){
    e.printStackTrace();
    return null;
}

多字段分组聚合

按fngroup和user_id分组聚合

public List selectModuleCount(Map param) {
    try(RestHighLevelClient client= elasticsearchConfig.getClient()) {

        // 1、创建search请求
        //SearchRequest searchRequest = new SearchRequest();
        SearchRequest searchRequest = new SearchRequest(INDEX_PAGEACCESS);

        // 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.size(0);

        //加入聚合
        //字段值项分组聚合
        TermsAggregationBuilder aggregation = AggregationBuilders.terms("by_fngroup")
                .script(new Script("doc['fngroup.keyword'] +'#'+doc['user_id']"))
                //.field("fngroup.keyword")
                .size(Integer.MAX_VALUE)
                .order(BucketOrder.aggregation("count", true));
        //计算每组的平均balance指标
        aggregation.subAggregation(AggregationBuilders.count("count")
                .field("sessionid"));
        sourceBuilder.aggregation(aggregation);

        searchRequest.source(sourceBuilder);

        //3、发送请求
        SearchResponse searchResponse = client.search(searchRequest,RequestOptions.DEFAULT);

        //4、处理响应
        //搜索结果状态信息
        List<Map> result=new ArrayList<>();
        if(RestStatus.OK.equals(searchResponse.status())) {
            // 获取聚合结果
            Aggregations aggregations = searchResponse.getAggregations();
            Terms byAgeAggregation = aggregations.get("by_fngroup");
            for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
                Map map=new HashMap();
                String[] arr= buck.getKeyAsString().split("#");
                map.put("module",arr[0].replace("[","").replace("]",""));
                map.put("user_id",arr[1].replace("[","").replace("]",""));
                //取子聚合
                ParsedValueCount averageBalance = buck.getAggregations().get("count");

                map.put("count",averageBalance.getValue());
                result.add(map);
            }
        }
        return result;
    }catch (Exception e){
        e.printStackTrace();
        return null;
    }

}