文章目录
一、业务工具类
elasticsearch查询数据的操作基本上都差不多，主要不同的是查询条件的组合。
1. 获取es连接（工具类）
public static RestHighLevelClient getEsHighInit() {
RestClientBuilder http = RestClient.builder(new HttpHost("192.168.248.10", 9200, "http"))
.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectTimeout(700000);
requestConfigBuilder.setSocketTimeout(600000);
requestConfigBuilder.setConnectionRequestTimeout(100000);
return requestConfigBuilder;
}
});
return new RestHighLevelClient(http);
}
2. 获取所有索引（工具类）
public static List<Map<String, Object>> getIndex(RestHighLevelClient esHighInit) throws IOException {
List<Map<String, Object>> resultList = new ArrayList();
GetAliasesRequest request = new GetAliasesRequest();
GetAliasesResponse alias = esHighInit.indices().getAlias(request, RequestOptions.DEFAULT);
Map<String, Set<AliasMetaData>> map = alias.getAliases();
map.forEach((k, v) -> {
if (!k.startsWith(".")) {//忽略elasticesearch 默认的
Map map1 = new HashMap();
map1.put("indexName", k);
resultList.add(map1);
}
});
return resultList;
}
3. 获取数据（工具类）
public static Map<String, Object> getData(RestHighLevelClient restHighLevelClient, String index, SearchSourceBuilder searchSourceBuilder) throws IOException {
Map<String, Object> result = new HashMap<>();
List<Map<String, Object>> datas = new ArrayList<>();
//1.创建 SearchRequest搜索请求
SearchRequest searchRequest = new SearchRequest();
//2.设置需要查看的索引
if (!StringUtils.isEmpty(index)) {
searchRequest.indices(index);//指定要查询的索引
}
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//4.执行查询
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
//5.解析查询结果
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
Map map = new HashMap();
map.put("index", hit.getIndex());//索引
map.put("source", hit.getSourceAsString());//内容
map.put("id", hit.getId());//内容
datas.add(map);
}
result.put("data", datas);
result.put("total", hits.getTotalHits().value);
return result;
}
4. 聚合查询数量（工具类）
public static List<Map<String, Object>> getGroupCount(RestHighLevelClient client, String index, SearchSourceBuilder sourceBuilder) throws IOException {
List<Map<String, Object>> result = new ArrayList<>();
//1.创建 SearchRequest搜索请求
SearchRequest searchRequest = new SearchRequest();
//2.设置需要查看的索引
if (!StringUtils.isEmpty(index)) {
searchRequest.indices(index);//指定要查询的索引
}
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(sourceBuilder);
//4.执行查询
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Terms terms = searchResponse.getAggregations().get("logCount");
//5.解析查询结果
List<? extends Terms.Bucket> groupBuckets = terms.getBuckets();
for (Terms.Bucket bucket : groupBuckets) {
Map<String, Object> data = new HashMap<>();
String key = bucket.getKeyAsString(); // 分组的key值
long count = bucket.getDocCount(); // 分组的日志数量
data.put("key", key);
data.put("count", count);
result.add(data);
}
return result;
}
5. 多字段聚合查询数量
//*******************************分组查询,实现聚合agent.hostname并且查询数量*******************************
//1.初始化连接
RestHighLevelClient esHighInit = getEsHighInitCommonConn();
//2.创建 SearchRequest搜索请求
SearchRequest searchRequest = new SearchRequest();
//3.设置分页和相关条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//返回结果不包含文档内容
searchSourceBuilder.size(0);
//设置聚合
//聚合字段1
TermsAggregationBuilder aggregationBuilder1 = AggregationBuilders.terms("group_by_field").field("application").size(10000);
CardinalityAggregationBuilder uniqueCountBuilder1 = AggregationBuilders.cardinality("unique_count").field("application");
aggregationBuilder1.subAggregation(uniqueCountBuilder1);
//聚合字段2
TermsAggregationBuilder aggregationBuilder2 = AggregationBuilders.terms("group_by_field").field("name").size(10000);
CardinalityAggregationBuilder uniqueCountBuilder2 = AggregationBuilders.cardinality("unique_count").field("name");
aggregationBuilder2.subAggregation(uniqueCountBuilder2);
aggregationBuilder1.subAggregation(aggregationBuilder2);
searchSourceBuilder.aggregation(aggregationBuilder1);
//4.设置索引
searchRequest.indices(".security-7");//指定要查询的索引
//5.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//6.执行查询
SearchResponse searchResponse = esHighInit.search(searchRequest, RequestOptions.DEFAULT);
//7.获取数据
Terms terms = searchResponse.getAggregations().get("group_by_field");
List<? extends Terms.Bucket> groupBuckets = terms.getBuckets();
List<Map<String, Object>> datas = new ArrayList<>();
//第一层聚合
for (Terms.Bucket bucket : groupBuckets) {
String application = bucket.getKeyAsString(); // 分组的key值
//第二层聚合
Terms terms1 =bucket.getAggregations().get("group_by_field");
List<? extends Terms.Bucket> groupBuckets1 = terms1.getBuckets();
for (Terms.Bucket bucket1 : groupBuckets1) {
String name = bucket1.getKeyAsString(); // 分组的key值
long count1 = bucket1.getDocCount(); // 分组的日志数量
Map map=new HashMap();
map.put("application",application);
map.put("name",name);
map.put("count",count1);
datas.add(map);
}
}
//8.关闭
esHighInit.close();
6. like查询（工具类）
//*******************************like查询，实现org*（org%）*******************************
//1.初始化连接
RestHighLevelClient esHighInit = getEsHighInitCommonConn();
//2.创建 SearchRequest搜索请求
SearchRequest searchRequest = new SearchRequest();
//3.设置分页和相关条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//设置分页,默认十条
searchSourceBuilder.from(0);
searchSourceBuilder.size(20);
//设置like过滤
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.wildcardQuery("message.keyword", "org*"));
searchSourceBuilder.query(boolQueryBuilder);
//4.设置索引
searchRequest.indices("hdfs-2023.07.18");//指定要查询的索引
//5.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//6.执行查询
SearchResponse searchResponse = esHighInit.search(searchRequest, RequestOptions.DEFAULT);
//7.获取数据
SearchHits hits = searchResponse.getHits();
long total=hits.getTotalHits().value;
SearchHit[] datas=hits.getHits();
//8.关闭
esHighInit.close();
7. 各种业务场景构造条件查询数据与统计
public static void main(String[] args) throws IOException {
//=======================获取es连接=======================
RestHighLevelClient esHighInit = getEsHighInit();
//=======================查询所有索引=======================
// List<Map<String, Object>> indexs = getIndex(esHighInit);
//=======================查询某个索引底下的所有数据=======================
//创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder1 = new SearchSourceBuilder();
// //查找所有
// MatchAllQueryBuilder matchAllQueryBuilder1 = QueryBuilders.matchAllQuery();
// searchSourceBuilder1.query(matchAllQueryBuilder1);
// List<String> messages1=matchAll(esHighInit, "index_test",searchSourceBuilder1);
//=======================查询某个索引底下的所有数据（根据条件）=======================
// //创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// //设置分页
// searchSourceBuilder.from(0);
// searchSourceBuilder.size(10);
// //设置数据顺序
// searchSourceBuilder.sort("count", SortOrder.ASC);
// //指定需要返回或者排除的字段
// String[] includes = {"count","address","age","name"};
// String[] excludes = {};
// searchSourceBuilder.fetchSource(includes, excludes);
// //查找所有
// //MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
// //searchSourceBuilder.query(matchAllQueryBuilder);
// //分词关键字查找
// MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("address", "天宫");
// matchQueryBuilder.operator(Operator.OR);
// searchSourceBuilder.query(matchQueryBuilder);
//
// List<String> messages=matchAll(esHighInit, "index_test",searchSourceBuilder);
//=======================查询某个索引底下的所有数据（精确查询Term）=======================
// //创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder();
// //Term查找,一般针对数字等，因为中文或者字母有些灰分分词，找不到，除非设置"index" : "not_analyzed"
// TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("count", 2);
// searchSourceBuilder2.query(termQueryBuilder);
// List<String> messages2=matchAll(esHighInit, "index_test",searchSourceBuilder2);
//=======================查询某个索引底下的所有数据=======================
//创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder1 = new SearchSourceBuilder();
// //查找所有
// MatchAllQueryBuilder matchAllQueryBuilder1 = QueryBuilders.matchAllQuery();
// searchSourceBuilder1.query(matchAllQueryBuilder1);
// List<String> messages1=matchAll(esHighInit, "index_test",searchSourceBuilder1);
//=======================查询某个索引底下的所有数据（日期查询range）=======================
// //创建 SearchSourceBuilder条件构造。
// SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder();
// //范围查询
// RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("count"); //date字段
// rangeQueryBuilder.lt(2);
// searchSourceBuilder3.query(rangeQueryBuilder);
// List<String> messages3 = matchAll(esHighInit, "index_test", searchSourceBuilder3);
//=======================查询某个索引底下的所有数据（多个id查询）=======================
// //创建 SearchSourceBuilder条件构造。
// SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder();
// //多个id 查找
// IdsQueryBuilder idsQueryBuilder4 = QueryBuilders.idsQuery();
// idsQueryBuilder4.addIds("15", "2", "3");
// searchSourceBuilder4.query(idsQueryBuilder4);
// List<String> messages3 = matchAll(esHighInit, "index_test", searchSourceBuilder4);
//=======================查询某个索引底下的所有数据（复合查询）=======================
// //创建 SearchSourceBuilder条件构造。
// SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder();
// //Bool查找
// BoolQueryBuilder boolQueryBuilder5 = QueryBuilders.boolQuery();
// // and
// //boolQueryBuilder5.must(QueryBuilders.rangeQuery("count").gte(2));
// //boolQueryBuilder5.must(QueryBuilders.matchQuery("address", "天"));
// // or
// boolQueryBuilder5.should(QueryBuilders.rangeQuery("count").gte(2));
// boolQueryBuilder5.should(QueryBuilders.matchQuery("address", "天"));
// searchSourceBuilder5.query(boolQueryBuilder5);
// List<String> messages3 = matchAll(esHighInit, "index_test", searchSourceBuilder5);
//=======================查询某个索引底下的所有数据（复合查询）=======================
// //创建 SearchSourceBuilder条件构造。
// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// //Fuzzy 查找
// FuzzyQueryBuilder fuzzyQueryBuilder = QueryBuilders.fuzzyQuery("address", "天").fuzziness(Fuzziness.ONE);
// searchSourceBuilder.query(fuzzyQueryBuilder);
// List<String> messages3 = matchAll(esHighInit, "index_test", searchSourceBuilder);
//=======================查询所有数据=======================
// //创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder1 = new SearchSourceBuilder();
// //设置分页,默认十条
// searchSourceBuilder1.from(0);
// searchSourceBuilder1.size(10);
// //查找所有
// MatchAllQueryBuilder matchAllQueryBuilder1 = QueryBuilders.matchAllQuery();
// searchSourceBuilder1.query(matchAllQueryBuilder1);
// Map<String, Object> result = matchAll(esHighInit, "", searchSourceBuilder1);
//*******************************查询真实日志文件数据(查询业务1)*******************************
// //创建 SearchSourceBuilder条件构造
// SearchSourceBuilder searchSourceBuilder1 = new SearchSourceBuilder();
// //设置分页,默认十条
// searchSourceBuilder1.from(0);
// searchSourceBuilder1.size(10);
// //filter过滤查询
// BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// boolQueryBuilder.must(QueryBuilders.rangeQuery("@timestamp").gte("2023-03-14 09:37:00").lte("2023-03-18 09:40:00").format("yyyy-MM-dd HH:mm:ss").timeZone("+08:00"));
// boolQueryBuilder.must(QueryBuilders.matchQuery("agent.hostname", "localhost.localdomain"));
// boolQueryBuilder.must(QueryBuilders.matchQuery("log.file.path", "/home/es/jar-log/txt.log"));
// boolQueryBuilder.must(QueryBuilders.matchQuery("message", "error"));
// boolQueryBuilder.must(QueryBuilders.matchQuery("fields.filetype.keyword", "jar-log"));
// searchSourceBuilder1.postFilter(boolQueryBuilder);
// Map<String, Object> result = getData(esHighInit, "", searchSourceBuilder1);
//*******************************查询真实日志文件数据(查询聚合数据2)*******************************
// SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
// TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("logCount").field("fields.filetype.keyword");
// sourceBuilder.aggregation(aggregationBuilder);
// List<Map<String, Object>> res = getGroupCount(esHighInit, "", sourceBuilder);
//*******************************查询真实日志文件数据(筛选条件并且聚合数据3)*******************************
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//filter过滤无效，这里只能使用query
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.termQuery("fields.filetype.keyword", "jar-log"));
boolQueryBuilder.must(QueryBuilders.termQuery("log.file.path.keyword", "/home/es/jar-log/txt.log"));
boolQueryBuilder.must(QueryBuilders.rangeQuery("@timestamp").gte("2023-03-14 09:37:00").lte("2023-03-18 09:40:00").format("yyyy-MM-dd HH:mm:ss").timeZone("+08:00"));
searchSourceBuilder.query(boolQueryBuilder);
//设置聚合字段
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("logCount").field("fields.filetype.keyword");
searchSourceBuilder.aggregation(aggregationBuilder);
//查询
List<Map<String, Object>> res = getGroupCount(esHighInit, "", searchSourceBuilder);
esHighInit.close();
}
二、不常用工具类
1. 获取某个字段最小、最大、平均值
public static void getdata1() throws IOException {
RestHighLevelClient restHighLevelClient = getEsHighInit();
//1.创建 SearchRequest搜索请求,并指定要查询的索引
SearchRequest searchRequest = new SearchRequest("index_test");
//2.创建 SearchSourceBuilder条件构造。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//查询员工的最低最高和平均工资
MaxAggregationBuilder maxAggregationBuilder = AggregationBuilders.max("maxSalary").field("count");
MinAggregationBuilder minAggregationBuilder = AggregationBuilders.min("minSalary").field("count");
AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("avgSalary").field("count");
searchSourceBuilder.aggregation(maxAggregationBuilder);
searchSourceBuilder.aggregation(minAggregationBuilder);
searchSourceBuilder.aggregation(avgAggregationBuilder);
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//4.执行查询
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
//5.解析查询结果
System.out.println(searchResponse);
System.out.println("花费的时长：" + searchResponse.getTook());
Aggregations aggregations = searchResponse.getAggregations();
System.out.println("aggregations：" + aggregations);
ParsedMax maxSalary = aggregations.get("maxSalary");
ParsedMin minSalary = aggregations.get("minSalary");
ParsedAvg avgSalary = aggregations.get("avgSalary");
System.out.println("maxSalary：" + maxSalary);
System.out.println("最小" + maxSalary.getValue());
System.out.println("最大" + minSalary.getValue());
System.out.println("平均" + avgSalary.getValue());
}
2. 获取某个字段最小、最大、平均值、个数、和
public static void getdata2() throws IOException {
RestHighLevelClient restHighLevelClient = getEsHighInit();
//1.创建 SearchRequest搜索请求,并指定要查询的索引
SearchRequest searchRequest = new SearchRequest("index_test");
//2.创建 SearchSourceBuilder条件构造。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//对salary进行统计
StatsAggregationBuilder statsAggregationBuilder = AggregationBuilders.stats("statSalary").field("count");
searchSourceBuilder.aggregation(statsAggregationBuilder);
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//4.执行查询
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
//5.解析查询结果
System.out.println("花费的时长：" + searchResponse.getTook());
Aggregations aggregations = searchResponse.getAggregations();
System.out.println("aggregations：" + aggregations);
ParsedStats statSalary = aggregations.get("statSalary");
System.out.println("统计个数" + statSalary.getCount());
System.out.println("统计平均" + statSalary.getAvg());
System.out.println("统计最高" + statSalary.getMaxAsString()); //可以转String
System.out.println("统计最低" + statSalary.getMin());
System.out.println("统计和" + statSalary.getSum());
}
3. 搜索结果去重统计
public static void getdata3() throws IOException {
RestHighLevelClient restHighLevelClient = getEsHighInit();
//1.创建 SearchRequest搜索请求,并指定要查询的索引
SearchRequest searchRequest = new SearchRequest("index_test");
//2.创建 SearchSourceBuilder条件构造。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//cardinate对搜索结果去重统计
CardinalityAggregationBuilder cardinalityAggregationBuilder = AggregationBuilders.cardinality("jobCardinate").field("count");
searchSourceBuilder.aggregation(cardinalityAggregationBuilder);
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//4.执行查询
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
//5.解析查询结果
System.out.println("花费的时长：" + searchResponse.getTook());
Aggregations aggregations = searchResponse.getAggregations();
System.out.println("aggregations：" + aggregations);
ParsedCardinality cardinality = aggregations.get("jobCardinate");
System.out.println("字段：" + cardinality.getName());
System.out.println("不重复的个数" + cardinality.getValue());
}
4. 按照字段的 Range 分桶
public static void getdata6() throws IOException {
RestHighLevelClient restHighLevelClient = getEsHighInit();
//1.创建 SearchRequest搜索请求,并指定要查询的索引
SearchRequest searchRequest = new SearchRequest("index_test");
//2.创建 SearchSourceBuilder条件构造。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// Salary Range分桶，可以自己定义 key
RangeAggregationBuilder rangeAggregationBuilder = AggregationBuilders.range("salary_range")
.field("count")
.addUnboundedTo(1)
.addRange(1, 2)
.addUnboundedFrom(">2", 20000);
searchSourceBuilder.aggregation(rangeAggregationBuilder);
//3.将 SearchSourceBuilder 添加到 SearchRequest中
searchRequest.source(searchSourceBuilder);
//4.执行查询
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
//5.解析查询结果
System.out.println("花费的时长：" + searchResponse.getTook());
SearchHits hits = searchResponse.getHits();
System.out.println("符合条件的总文档数量：" + hits.getTotalHits().value);
//聚合信息
Aggregations aggregations = searchResponse.getAggregations();
System.out.println("aggregations：" + aggregations);
ParsedRange salaryRange = aggregations.get("salary_range"); //注意类型
System.out.println("字段：" + salaryRange.getName());
List<? extends Range.Bucket> buckets = salaryRange.getBuckets();
for (Range.Bucket bucket : buckets) {
System.out.println("getKey：" + bucket.getKey());
System.out.println("getKeyAsString：" + bucket.getKeyAsString());
System.out.println("getDocCount：" + bucket.getDocCount());
System.out.println("getFromAsString：" + bucket.getFromAsString());
System.out.println("getToAsString：" + bucket.getToAsString());
}
}
1164
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
