elasticsearch + java实现复杂查询
首先项目中需要集成elasticsearch,如不会集成请参考《集成地址》,如不会安装elasticsearch请参考《安装地址》
文章目录
前言
按照现阶段项目的要求,简单的查询已不满足现阶段。这方面的资料在全网都非常少, 接下来下面会罗列出比如汇总查询,must、should filter组合查询等。
一、分组统计查询
注意:分组如果在字符串字段上,需要建立字段对应的.keyword字段,该字段支持聚合处理,直接用字符串字段会报错。
1.根据src_ip单个字段做分组
@Autowired
private RestHighLevelClient esClient;
@Test
void groupBy() {
try {
SearchRequest searchRequest = new SearchRequest("sh.security-*");
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("src_ip").field("src_ip" + ".keyword").size(100000);
sourceBuilder.aggregation(aggregationBuilder);
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);
if (RestStatus.OK.equals(searchResponse.status())) {
// 获取聚合结果
Aggregations aggregations = searchResponse.getAggregations();
Terms byAgeAggregation = aggregations.get("src_ip");
for (Terms.Bucket buck : byAgeAggregation.getBuckets()) {
System.out.println("key: " + buck.getKeyAsString());
System.out.println("docCount: " + buck.getDocCount());
System.out.println("docCountError: " + buck.getDocCountError());
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
2.根据src_ip和src_port多个字段进行分组
@Test
void MultipleGroupBy(){
try {
// 1、创建search请求
SearchRequest searchRequest = new SearchRequest("sh.security-*");
// 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法,构造各种查询的方法都在这。
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.size(0);
//加入聚合
//字段值项分组聚合
TermsAggregationBuilder aggregation = AggregationBuilders.terms("src_ip")
.script(new Script("doc['src_ip.keyword'] +'#'+doc['src_port.keyword']"))
//.field("src_ip.keyword")
.size(Integer.MAX_VALUE);
sourceBuilder.aggregation(aggregation);
searchRequest.source(sourceBuilder);
//3、发送请求
SearchResponse searchResponse = esClient.search(searchRequest,RequestOptions.DEFAULT);
//4、处理响应
//搜索结果状态信息
if(RestStatus.OK.equals(searchResponse.status())) {
// 获取聚合结果
Aggregations aggregations = searchResponse.getAggregations();
Terms byAgeAggregation = aggregations.get("src_ip");
for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
System.out.println("key: " + buck.getKeyAsString());
System.out.println("docCount: " + buck.getDocCount());
System.out.println("docCountError: " + buck.getDocCountError());
}
}
}catch (Exception e){
e.printStackTrace();
}
}
二、echart统计数据
根据时间统计
@Test
void chart(){
try {
String startTimeStr = "2022-08-01 00:00:00";
String endTimeStr = "2022-08-31 23:59:59";
// 判断索引是否为空
SearchRequest request = new SearchRequest("sh.security-*");
// 开启真实总数
request.source().trackTotalHits(true);
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
// 时间范围
long startTime = DateUtils.parseTimezone(startTimeStr)*1000L;
long endTime = DateUtils.parseTimezone(endTimeStr)*1000L;
boolQuery.must(QueryBuilders.rangeQuery("occur_time").gt(startTime).lt(endTime));
// 间隔时间
DateHistogramInterval dateHistogramInterval = DateHistogramInterval.days(1);
AggregationBuilder agg = AggregationBuilders.dateHistogram("agg_by_time")
.dateHistogramInterval(dateHistogramInterval)
.field("occur_time")
.minDocCount(0);
// 请求
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(boolQuery)
.aggregation(agg)
.size(0);
request.source(searchSourceBuilder);
SearchResponse response = esClient.search(request, RequestOptions.DEFAULT);
Aggregations aggregations = response.getAggregations();
ParsedDateHistogram terms = aggregations.get("agg_by_time");
for (Histogram.Bucket bucket : terms.getBuckets()) {
System.out.println(DateUtils.getFormatDate(Long.parseLong(bucket.getKeyAsString())));
System.out.println(bucket.getDocCount());
}
}catch (IOException e){
throw new RuntimeException(e);
}
}