spring data es高级特性基本用法

一、获取ES客户端

• 根据spring官方介绍，推荐使用RestHighLevelClient、ElasticsearchRestTemplate进行ES的CRUD包括聚合操作
• spring data ES官方：https://docs.spring.io/spring-data/elasticsearch/docs/4.1.2/reference/html/#elasticsearch.clients.rest

@Data
@Configuration
@ConfigurationProperties(prefix = "custom.params.es")
public class RestHighLevelClientConfig extends AbstractElasticsearchConfiguration {

    private String host;

    private Integer port;

    private String username;

    private String password;
	//获取RestHighLevelClient客户端对象
    @Bean
    public RestHighLevelClient elasticsearchClient() {
        BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
        		//若ES连接需要用户名密码，通过BasicCredentialsProvider来设置用户密码连接
                new UsernamePasswordCredentials(username, password));

        RestHighLevelClient esClient = new RestHighLevelClient(
        		//host为连接地址，port为ES服务端口号
                RestClient.builder(new HttpHost(host, port))
                        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                                httpClientBuilder.disableAuthCaching();
                                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                            }
                        }));
        return esClient;
    }
	//获取ElasticsearchRestTemplate模板对象
    @Bean
    public ElasticsearchRestTemplate elasticsearchRestTemplate(RestHighLevelClient client){
        return new ElasticsearchRestTemplate(client);
    }

}

二、条件查询

//设置查询条件，等同于
/*
"query": {
    "bool": {
      "must": [
       ....
      ],
*/
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); //获取查询构造器
//.matchQuery()模糊匹配查询, .rangQuery()范围查询，.termQuery()精准查询
queryBuilder.must(QueryBuilders.termQuery(RECORD_ID, id.toString())) //设置查询条件为term

三、聚合

//ES查询聚合体语法部分
  "aggregations": {
    "agg_timestamp_minute": {
      "date_histogram": {
        "field": "timestamp",  //根据timestamp字段查询每分钟的聚合
        "interval": "1m", 
        "offset": 0,
        "order": {
          "_key": "asc"		//顺序排列，默认
        },
        "keyed": false,
        "min_doc_count": 0
      },
      "aggregations": {
        "agg_sum_response_time": {
          "sum": {
            "field": "responseTime"  //sum求该字段聚合后的和
          }
        }
      }
    }
  }

//FieldConstant命名的字段常量
public class FieldConstant {

    public static final String AGG_TOTAL = "agg_total";

    public static final String AGG_TIMESTAMP = "agg_timestamp";

    public static final String AGG_TIMESTAMP_MINUTE = "agg_timestamp_minute";

    public static final String AGG_SUM_RESPONSE_TIME = "agg_sum_response_time";

    public static final String AGG_SUM_ERROR_COUNT = "agg_sum_error_count";

    public static final String AGG_SUM_COUNT = "agg_sum_count";

    public static final String AVG_TPS = "avg_tps";

    public static final String AVG_USERS = "avg_users";

    public static final String AGG_SUM_USERS = "agg_sum_users";

    public static final String AVG_RESPONSE_TIME = "avg_response_time";

    public static final String MAX_METRIC_THREADS = "max_all_threads";

    public static final String MAX_RESPONSE_TIME = "max_response_time";

    public static final String MIN_RESPONSE_TIME = "min_response_time";

    public static final String METRIC_TIMESTAMP = "timestamp";

    public static final String METRIC_RESPONSE_TIME = "responseTime";

    public static final String METRIC_THREADS = "allThreads";

    public static final String METRIC_ERROR_COUNT = "errorCount";

}

//AggregationBuilders获取聚合条件构造器
       AggregationBuilder aggregation = AggregationBuilders
       			//dateHistogram为按时间方式进行聚合，
                .dateHistogram(FieldConstant.AGG_TIMESTAMP_MINUTE)
                .field(FieldConstant.METRIC_TIMESTAMP)
                //聚合条件为每分钟
                .interval(1)
                .dateHistogramInterval(DateHistogramInterval.MINUTE)    
//FieldConstant.AGG_SUM_RESPONSE_TIME 为sum聚合后命名的字段名，  FieldConstant.METRIC_RESPONSE_TIME为聚合的目标字段              
                .subAggregation(AggregationBuilders.sum(FieldConstant.AGG_SUM_RESPONSE_TIME)
                        .field(FieldConstant.METRIC_RESPONSE_TIME)) 
                .subAggregation(AggregationBuilders.max(FieldConstant.MAX_METRIC_THREADS)
                        .field(FieldConstant.METRIC_THREADS))
                .subAggregation(AggregationBuilders.sum(FieldConstant.AGG_SUM_USERS)
                        .field(FieldConstant.METRIC_THREADS))
                .subAggregation(AggregationBuilders.avg(FieldConstant.AVG_RESPONSE_TIME)
                        .field(FieldConstant.METRIC_RESPONSE_TIME));

为方便理解将ES聚合转为SQL方式可参考该作者文章：https://www.cnblogs.com/xionggeclub/p/7975982.html

四、聚合+查询构成整体

//获取ES客户端对象
@Resource
private RestHighLevelClient restHighLevelClient;

//将查询聚合构造为整体
SearchSourceBuilder builder = new SearchSourceBuilder()
        .query(queryBuilder)  //条件查询部分
        .aggregation(aggregation)  // 聚合部分
        .size(0); //查询到的数据展示数量，0不展示
        
//设置查询的索引，等同于GET jmeter-metric-*/_search
SearchRequest searchRequest = new SearchRequest(INDEX);  // INDEX="jmeter-metric-*"
searchRequest.source(builder);

//restHighLevelClient为ES客户端高级用法，response为获取到的查询返回内容
SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); 

//获取到response数据体中查询条件命中的数据，等同于下图1
SearchHit[] hits = response.getHits().getHits();

//获取到聚合的桶数据，等同于下图2
ParsedDateHistogram bucket = response.getAggregations().get(FieldConstant.AGG_TIMESTAMP_MINUTE)

图1

图2

五、使用ElasticsearchRestTemplate来插入数据

• 通过kafka消费数据来写入ES

@Service
@Slf4j
public class KafkaConsumer {

    private static final String INDEX = "jmeter-metric-";

    @Resource
    private ElasticsearchRestTemplate elasticsearchRestTemplate;

    @KafkaListener(topics = "${custom.params.kafka.topic}", groupId = "${spring.kafka.consumer.properties.group.id}")
    public void receiveMessage(ConsumerRecord<?, ?> record) {
    	//将消费的数据，映射到JmeterMetricData对象
        JmeterMetricData datas = JSON.parseObject(record.value().toString(), JmeterMetricData.class);
        log.info("jmeter_metric_topic：" + JSON.toJSONString(datas));
        String formatDate = DateUtil.formatDate();
        //IndexQuery设置数据
        IndexQuery indexQuery = new IndexQuery();
        indexQuery.setObject(datas);
        //IndexCoordinates来初始化索引
        IndexCoordinates indexCoordinates = IndexCoordinates.of(INDEX + formatDate);
        //插入
        elasticsearchRestTemplate.index(indexQuery, indexCoordinates);
    }
}