平铺httpClient获取的es聚合的数据
目录
学习了 ，使用平铺 RestHighLevelClient 获取到es聚合的数据，如果使用commons-httpclient的数据如何平铺呢？
处理思路：
在处理值的时候，思路差不多。先处理简单的，再处理复杂的，逐层解析。把常见的例子都请求一遍。HttpClient处理的情况会少一些。
代码：
相关引用和参数类参考：
1，值的处理
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.math.BigDecimal;
import java.util.*;
public class EsParseStrUtils {
public static final String KEY = "key";
public static final String VALUE = "value";
public static final String COUNT = "doc_count";
public static final String BUCKETS = "buckets";
public static final String AGG = "aggregations";
public static List<Map<String, Object>> getEsBuckets(JSONObject esData){
return Optional.ofNullable(esData).map(e -> e.getJSONObject(AGG))
.map(agg -> {
//获取buckets的值
List<Map<String, Object>> bucketList = getDeepNextAggs(agg);
// 处理只有value的情况
if (CollectionUtils.isEmpty(bucketList)) {
// sum,avg等
Map<String, Object> valueMap = getValueMap(agg);
bucketList.add(valueMap);
}
return bucketList;
}).orElse(new ArrayList<>());
}
// 获取 带value的值
public static Map<String, Object> getValueMap(Map<String, Object> aggMap){
Map<String, Object> valueMap = new HashMap<>();
MapUtils.emptyIfNull(aggMap).forEach((key2, value2) -> {
String valueStr2 = value2.toString();
if (valueStr2.contains("{")) {
Map<String, Object> innerValueMap = (Map<String, Object>) value2;
Object value3 = innerValueMap.get(VALUE);
if (value3 != null) {
String valueStr3 = value3.toString();
if(judgeNumber(valueStr3)){
valueMap.put(key2, formatDouble(Double.parseDouble(valueStr3)));
}else{
valueMap.put(key2, value3);
}
}else{
valueMap.putAll(innerValueMap);
}
}
});
return valueMap;
}
/**
* 判断字符串是否为浮点型
*/
public static boolean judgeNumber(String str) {
if (StringUtils.isNotEmpty(str)) {
String reg = "^[0-9]+(.[0-9]+)?$";
return str.matches(reg);
}
return false;
}
private static double formatDouble(double value) {
BigDecimal bigDecimal = new BigDecimal(value);
value = bigDecimal.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue();
return value;
}
// 处理 bucket里面的值；
public static List<Map<String, Object>> getDeepNextAggs(Map<String, Object> agg){
List<Map<String, Object>> bucketList = new ArrayList<>();
MapUtils.emptyIfNull(agg).forEach((key, value1) -> {
String valueStr1 = value1.toString();
if(valueStr1.contains("{")){
Map<String, Object> aggValueMap = (Map<String, Object>) value1;
Object buckets = aggValueMap.get(BUCKETS);
if (buckets != null) {
List<Map<String, Object>> bucktList = (List<Map<String, Object>>) buckets;
ListUtils.emptyIfNull(bucktList).forEach(f -> {
Map<String, Object> bucketMap = new HashMap<>();
bucketMap.put(key, MapUtils.getString(f, KEY));
bucketMap.put(key+"."+COUNT, MapUtils.getString(f, COUNT));
String fromString = MapUtils.getString(f, "from_as_string");
if(StringUtils.isNotEmpty(fromString)){
bucketMap.put("from", fromString);
}
String toString = MapUtils.getString(f, "to_as_string");
if(StringUtils.isNotEmpty(toString)){
bucketMap.put("to", toString);
}
String keyString = MapUtils.getString(f, "key_as_string");
if(StringUtils.isNotEmpty(keyString)){
bucketMap.put(key, keyString);
}
// 获取 带value的值
Map<String, Object> valueMap = getValueMap(f);
bucketMap.putAll(valueMap);
boolean containBucket = false;
// 循环调用获取bucket的值
List<Map<String, Object>> deepAggs = getDeepNextAggs(f);
if(CollectionUtils.isNotEmpty(deepAggs)){
ListUtils.emptyIfNull(deepAggs).forEach(e -> {
e.putAll(bucketMap);
bucketList.add(e);
});
containBucket = true;
}
if(!containBucket){
bucketList.add(bucketMap);
}
});
}
}
});
return bucketList;
}
}值类型处理部分先处理常见的部分，后面再根据需要增加补充完善
2，模拟数据
public static JSONObject initDeepBucketFourth(){
String data = "{\n" +
" \"took\" : 11,\n" +
" \"timed_out\" : false,\n" +
" \"_shards\" : {\n" +
" \"total\" : 5,\n" +
" \"successful\" : 5,\n" +
" \"skipped\" : 0,\n" +
" \"failed\" : 0\n" +
" },\n" +
" \"hits\" : {\n" +
" \"total\" : 8,\n" +
" \"max_score\" : 0.0,\n" +
" \"hits\" : [ ]\n" +
" },\n" +
" \"aggregations\" : {\n" +
" \"colors\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"red\",\n" +
" \"doc_count\" : 4,\n" +
" \"avg_price\" : {\n" +
" \"value\" : 32500.0\n" +
" },\n" +
" \"make\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"honda\",\n" +
" \"doc_count\" : 3,\n" +
" \"max_price\" : {\n" +
" \"value\" : 20000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 10000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"jinshu\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 1,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 1.0\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"lvban\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 2,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 2.0\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"tiekuai\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 3,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 3.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"bmw\",\n" +
" \"doc_count\" : 1,\n" +
" \"max_price\" : {\n" +
" \"value\" : 80000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 80000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"jinshu\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 1,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 1.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"blue\",\n" +
" \"doc_count\" : 2,\n" +
" \"avg_price\" : {\n" +
" \"value\" : 20000.0\n" +
" },\n" +
" \"make\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"ford\",\n" +
" \"doc_count\" : 1,\n" +
" \"max_price\" : {\n" +
" \"value\" : 25000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 25000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"tiekuai\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 3,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 3.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"toyota\",\n" +
" \"doc_count\" : 1,\n" +
" \"max_price\" : {\n" +
" \"value\" : 15000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 15000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"lvban\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 2,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 2.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"green\",\n" +
" \"doc_count\" : 2,\n" +
" \"avg_price\" : {\n" +
" \"value\" : 21000.0\n" +
" },\n" +
" \"make\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"ford\",\n" +
" \"doc_count\" : 1,\n" +
" \"max_price\" : {\n" +
" \"value\" : 30000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 30000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"tiekuai\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 3,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 3.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" },\n" +
" {\n" +
" \"key\" : \"toyota\",\n" +
" \"doc_count\" : 1,\n" +
" \"max_price\" : {\n" +
" \"value\" : 12000.0\n" +
" },\n" +
" \"min_price\" : {\n" +
" \"value\" : 12000.0\n" +
" },\n" +
" \"materials\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : \"jinshu\",\n" +
" \"doc_count\" : 1,\n" +
" \"levels\" : {\n" +
" \"doc_count_error_upper_bound\" : 0,\n" +
" \"sum_other_doc_count\" : 0,\n" +
" \"buckets\" : [\n" +
" {\n" +
" \"key\" : 1,\n" +
" \"doc_count\" : 1\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"avg_level\" : {\n" +
" \"value\" : 1.0\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" }\n" +
" ]\n" +
" }\n" +
" }\n" +
"}\n";
JSONObject dataJson = JSONObject.parseObject(data);
System.out.println(dataJson);
return dataJson;
}
这边用不断嵌套的数据，作为例子
3，结果：
1，请求：
public static void main(String[] args) {
JSONObject esData = initDeepBucketFourth();
List<Map<String, Object>> lists = EsParseStrUtils.getEsBuckets(esData);
System.out.println("================= lists ================ ");
lists.forEach(System.out::println);
}
2,平铺的结果：
{"_shards":{"total":5,"failed":0,"successful":5,"skipped":0},"hits":{"hits":[],"total":8,"max_score":0.0},"took":11,"timed_out":false,"aggregations":{"colors":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":4,"avg_price":{"value":32500.0},"make":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":3,"max_price":{"value":20000.0},"min_price":{"value":10000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"jinshu","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":1}]},"avg_level":{"value":1.0}},{"doc_count":1,"key":"lvban","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":2}]},"avg_level":{"value":2.0}},{"doc_count":1,"key":"tiekuai","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":3}]},"avg_level":{"value":3.0}}]},"key":"honda"},{"doc_count":1,"max_price":{"value":80000.0},"min_price":{"value":80000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"jinshu","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":1}]},"avg_level":{"value":1.0}}]},"key":"bmw"}]},"key":"red"},{"doc_count":2,"avg_price":{"value":20000.0},"make":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"max_price":{"value":25000.0},"min_price":{"value":25000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"tiekuai","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":3}]},"avg_level":{"value":3.0}}]},"key":"ford"},{"doc_count":1,"max_price":{"value":15000.0},"min_price":{"value":15000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"lvban","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":2}]},"avg_level":{"value":2.0}}]},"key":"toyota"}]},"key":"blue"},{"doc_count":2,"avg_price":{"value":21000.0},"make":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"max_price":{"value":30000.0},"min_price":{"value":30000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"tiekuai","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":3}]},"avg_level":{"value":3.0}}]},"key":"ford"},{"doc_count":1,"max_price":{"value":12000.0},"min_price":{"value":12000.0},"materials":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":"jinshu","levels":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"doc_count":1,"key":1}]},"avg_level":{"value":1.0}}]},"key":"toyota"}]},"key":"green"}]}}}
================= lists ================
{materials.doc_count=1, make.doc_count=3, max_price=20000.0, min_price=10000.0, materials=jinshu, levels.doc_count=1, avg_price=32500.0, colors.doc_count=4, make=honda, levels=1, colors=red, avg_level=1.0}
{materials.doc_count=1, make.doc_count=3, max_price=20000.0, min_price=10000.0, materials=lvban, levels.doc_count=1, avg_price=32500.0, colors.doc_count=4, make=honda, levels=2, colors=red, avg_level=2.0}
{materials.doc_count=1, make.doc_count=3, max_price=20000.0, min_price=10000.0, materials=tiekuai, levels.doc_count=1, avg_price=32500.0, colors.doc_count=4, make=honda, levels=3, colors=red, avg_level=3.0}
{materials.doc_count=1, make.doc_count=1, max_price=80000.0, min_price=80000.0, materials=jinshu, levels.doc_count=1, avg_price=32500.0, colors.doc_count=4, make=bmw, levels=1, colors=red, avg_level=1.0}
{materials.doc_count=1, make.doc_count=1, max_price=25000.0, min_price=25000.0, materials=tiekuai, levels.doc_count=1, avg_price=20000.0, colors.doc_count=2, make=ford, levels=3, colors=blue, avg_level=3.0}
{materials.doc_count=1, make.doc_count=1, max_price=15000.0, min_price=15000.0, materials=lvban, levels.doc_count=1, avg_price=20000.0, colors.doc_count=2, make=toyota, levels=2, colors=blue, avg_level=2.0}
{materials.doc_count=1, make.doc_count=1, max_price=30000.0, min_price=30000.0, materials=tiekuai, levels.doc_count=1, avg_price=21000.0, colors.doc_count=2, make=ford, levels=3, colors=green, avg_level=3.0}
{materials.doc_count=1, make.doc_count=1, max_price=12000.0, min_price=12000.0, materials=jinshu, levels.doc_count=1, avg_price=21000.0, colors.doc_count=2, make=toyota, levels=1, colors=green, avg_level=1.0}总结：
平铺commons-httpclient获取Es聚合的值，处理起来比起RestHighLevelClient相对会简单一些，没有那么多类型的对应，处理浮点型数据的时候，要进行判断处理。
489
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
