1. [代码]spring配置文件
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"
default-autowire="byName">
<bean id="cateEsClientFactory" class="com.jd.pop.odp.util.esdb.EsClientFactory">
<constructor-arg name="clusterName" value=""/>
<constructor-arg name="ips">
<value></value>
</constructor-arg>
<constructor-arg name="esPort" value=""/>
</bean>
<bean id="realTimeEsClientFactory" class="com.jd.pop.odp.util.esdb.EsClientFactory">
<constructor-arg name="clusterName" value=""/>
<constructor-arg name="ips">
<value></value>
</constructor-arg>
<constructor-arg name="esPort" value=""/>
</bean>
<bean id="cateEsDao" class="com.jd.pop.es.dao.cate.impl.CateEsDaoImpl">
<property name="esClientFactory" ref="cateEsClientFactory"/>
</bean>
<bean id="realTimeEsDao" class="com.jd.pop.es.dao.realTime.impl.RealTimeEsDaoImpl">
<property name="esClientFactory" ref="realTimeEsClientFactory"/>
</bean>
</beans>
2. [代码]ES工厂类
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
public class EsClientFactory {
private static final Log logger = LogFactory.getLog(EsClientFactory.class);
// 是否扫描集群
private static boolean sniff = false;
// ES 集群名称
private static String clusterName;
// IP地址
private static String[] ips;
// 端口
private static int esPort;
private TransportClient esClient;//ES 客户端对象
public EsClientFactory(String clusterName,String[] ips,int esPort) {
this.clusterName=clusterName;
this.ips=ips;
this.esPort=esPort;
init();
}
/**
* ES 客户端连接初始化
*
* @return ES客户端对象
*/
private void init() {
Preconditions.checkNotNull(clusterName, "es 服务clusterName未配置");
Preconditions.checkNotNull(ips, "es 服务ip未配置");
Preconditions.checkArgument(esPort > 0, "es 服务服务port未配置");
//设置集群的名字
Settings settings = ImmutableSettings.settingsBuilder()
.put("cluster.name", clusterName)
.put("client.transport.sniff", sniff)
.build();
//创建集群client并添加集群节点地址
esClient = new TransportClient(settings);
for (String ip : ips) {
esClient.addTransportAddress(new InetSocketTransportAddress(ip, esPort));
}
}
public TransportClient getEsClient() {
return esClient;
}
public boolean isSniff() {
return sniff;
}
public String getClusterName() {
return clusterName;
}
public String[] getIps() {
return ips;
}
public int getEsPort() {
return esPort;
}
}
3. [代码]dao层调用
public class CateEsDaoImpl extends EsDbUtils implements CateEsDao {
@Override
public List<CateVenderData> queryListByFilterQuery(EsQueryObj esQueryObj) throws Exception {
return (List<CateVenderData>)queryObjectListByFilterQuery(esQueryObj,CateVenderData.class);
}
@Override
public List<CateVenderData> queryListByFilterQueryWithAgg(EsQueryObj esQueryObj) throws Exception {
return (List<CateVenderData>)queryObjectListByFilterQueryWithAgg(esQueryObj,CateVenderData.class);
}
}
public class RealTimeEsDaoImpl extends EsDbUtils implements RealTimeEsDao {
@Override
public List<OdpOperatorSum> queryListByFilterQuery(EsQueryObj esQueryObj) throws Exception {
return (List<OdpOperatorSum>)queryObjectListByFilterQuery(esQueryObj,OdpOperatorSum.class);
}
@Override
public List<OdpOperatorSum> queryListByFilterQueryWithAgg(EsQueryObj esQueryObj) throws Exception {
return (List<OdpOperatorSum>)queryObjectListByFilterQueryWithAgg(esQueryObj,OdpOperatorSum.class);
}
}
4. [代码]核心工具类
import com.google.gson.Gson;
import org.apache.commons.beanutils.PropertyUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.util.*;
/**
* Created with IntelliJ IDEA.
* User: supeidong
* Date: 2016/3/16
* Time: 13:37
* To change this template use File | Settings | File Templates.
*/
public class EsDbUtils <T>{
private static final Logger logger = LoggerFactory.getLogger(EsDbUtils.class);
private static final int FromIndex = 0;
private static final int MinSize = 100;
private static final int MaxSize = 100000;
private static final int GroupMinSize = 100;
private static final int GroupMaxSize = 500;
private EsClientFactory esClientFactory;//ES 客户端工厂类
/**
* 根据过滤条件查询出数据列表.需要传递索引和表名
* @param esQueryObj ES查询对象
* @param targetClass ES结果需要转换的类型
* @return
*/
public List<? extends Object> queryObjectListByFilterQuery(EsQueryObj esQueryObj,Class targetClass) throws Exception {
validationEsQuery(esQueryObj);
List<Object> esRecords = new ArrayList<Object>();
long startCountTime = System.currentTimeMillis();
//创建ES查询Request对象
SearchRequestBuilder esSearch= esClientFactory.getEsClient().prepareSearch(esQueryObj.getIndexName());
esSearch.setTypes(esQueryObj.getTypeName())
.setSearchType(SearchType.QUERY_THEN_FETCH)
.setFrom((esQueryObj.getFromIndex() > 0) ? esQueryObj.getFromIndex() : FromIndex)
.setSize((0 <esQueryObj.getSize() && esQueryObj.getSize() <= MaxSize) ? esQueryObj.getSize() : MinSize);
//添加查询条件
if(esQueryObj.getAndFilterBuilder()!=null){
esSearch.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), esQueryObj.getAndFilterBuilder()));
}
//添加多级排序
if(esQueryObj.getSortField()!=null) {
for (Map.Entry<String, SortOrder> entry : esQueryObj.getSortField().entrySet()) {
esSearch.addSort(entry.getKey(), entry.getValue());
}
}
//执行查询
SearchResponse response =esSearch .execute().actionGet();
for (SearchHit hit : response.getHits()) {
Object t = mapResult(hit.sourceAsMap(), targetClass);
esRecords.add(t);
}
logger.info("queryObjectListByFilterQuery search " + response.getHits().getTotalHits() + " data ,esQueryObj=" + new Gson().toJson(esQueryObj)+"-----------------------------------------! use " + (System.currentTimeMillis() - startCountTime) + " ms.");
return esRecords;
}
/**
* 根据过滤条件和分组键SUM/AVG键获取分组结果(目前分组结果不支持LIMIT操作)
* @param esQueryObj ES查询对象
* @param targetClass ES结果需要转换的类型
* @return
* @throws Exception
*/
public List<? extends Object> queryObjectListByFilterQueryWithAgg(EsQueryObj esQueryObj,Class targetClass) throws Exception {
validationEsGroupQuery(esQueryObj);
List<Object> esRecords = new ArrayList<Object>();
long startCountTime = System.currentTimeMillis();
if( esQueryObj.getSumFields()==null){
esQueryObj.setSumFields(new ArrayList<String>());
}
if(esQueryObj.getAvgFields()==null){
esQueryObj.setAvgFields(new ArrayList<String>());
}
TermsBuilder agg = getEsAgg(esQueryObj);
//创建ES查询Request对象
SearchRequestBuilder esSearch= esClientFactory.getEsClient().prepareSearch(esQueryObj.getIndexName());
esSearch.setTypes(esQueryObj.getTypeName())
.setSearchType(SearchType.QUERY_THEN_FETCH)
.addAggregation(agg);
//添加查询条件
if(esQueryObj.getAndFilterBuilder()!=null){
esSearch.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), esQueryObj.getAndFilterBuilder()));
}
//添加多级排序
if(esQueryObj.getSortField()!=null) {
for (Map.Entry<String, SortOrder> entry : esQueryObj.getSortField().entrySet()) {
esSearch.addSort(entry.getKey(), entry.getValue());
}
}
//执行查询
SearchResponse response =esSearch .execute().actionGet();
List<Map<String, Object>> aggMaps= getAggMap(response,esQueryObj.getGroupFields(), esQueryObj.getSumFields(), esQueryObj.getAvgFields());
for(Map<String, Object> aggMap : aggMaps){
Object t = mapResult(aggMap, targetClass);
esRecords.add(t);
}
logger.info("queryObjectListByFilterQuery search " + response.getHits().getTotalHits() + " data,esQueryObj=" + new Gson().toJson(esQueryObj)+"-----------------------------------------! use " + (System.currentTimeMillis() - startCountTime) + " ms.");
return esRecords;
}
/**
* 根据分组键和SUM/AVG键组合AGG条件
* @param esQueryObj
* @return
*/
private TermsBuilder getEsAgg(EsQueryObj esQueryObj) throws Exception{
List<String> groupFields= esQueryObj.getGroupFields();
List<String> sumFields= esQueryObj.getSumFields();
List<String> avgFields= esQueryObj.getAvgFields();
int groupSize=esQueryObj.getGroupSize();
Map<String, SortOrder> groupSortMap=esQueryObj.getGroupSortField();
TermsBuilder termsBuilder = AggregationBuilders.terms(groupFields.get(0)).field(groupFields.get(0));
if (groupFields.size() == 1) {
//设置排序后最后一层的结果数目
termsBuilder.size((0 <groupSize && groupSize <= GroupMaxSize) ? groupSize : GroupMinSize);
//添加group排序字段
if(groupSortMap!=null) {
List<Terms.Order> termsOrders=new ArrayList<Terms.Order>();
for (Map.Entry<String, SortOrder> entry : groupSortMap.entrySet()) {
if(entry.getValue().equals(SortOrder.ASC)){
termsOrders.add(Terms.Order.aggregation(entry.getKey(),true));
}else{
termsOrders.add(Terms.Order.aggregation(entry.getKey(), false));
}
}
termsBuilder.order(Terms.Order.compound(termsOrders));
}
for (String avgField : avgFields) {
termsBuilder.subAggregation(AggregationBuilders.avg(avgField).field(avgField));
}
for (String sumField : sumFields) {
termsBuilder.subAggregation(AggregationBuilders.sum(sumField).field(sumField));
}
} else {
termsBuilder.subAggregation(getChildTermsBuilder(groupFields, 1, sumFields, avgFields,groupSize,groupSortMap));
//设置最外层分组量
termsBuilder.size(GroupMaxSize);
}
return termsBuilder;
}
/**
* 通过递归的方式获取bucket agg分组语句
* @param groupFields
* @param i
* @param sumFields
* @param avgFields
* @return
*/
private TermsBuilder getChildTermsBuilder(List<String> groupFields,int i,List<String> sumFields, List<String> avgFields,int groupSize,Map<String, SortOrder> groupSortMap){
if(i+1==groupFields.size()){
TermsBuilder termsBuilderLast = AggregationBuilders.terms(groupFields.get(i)).field(groupFields.get(i));
//设置排序后最后一层的结果数目
termsBuilderLast.size((0 <groupSize && groupSize <= GroupMaxSize) ? groupSize : GroupMinSize);
//添加group排序字段
if(groupSortMap!=null) {
for (Map.Entry<String, SortOrder> entry : groupSortMap.entrySet()) {
if(entry.getValue().equals(SortOrder.ASC)){
termsBuilderLast.order(Terms.Order.aggregation(entry.getKey(),true));
}else{
termsBuilderLast.order(Terms.Order.aggregation(entry.getKey(),false));
}
}
}
for (String avgField : avgFields) {
termsBuilderLast.subAggregation(AggregationBuilders.avg(avgField).field(avgField));
}
for (String sumField : sumFields) {
termsBuilderLast.subAggregation(AggregationBuilders.sum(sumField).field(sumField));
}
return termsBuilderLast;
}
else{
TermsBuilder termsBuilder= AggregationBuilders.terms(groupFields.get(i)).field(groupFields.get(i));
//设置最外层分组量
termsBuilder.size(GroupMaxSize);
return termsBuilder.subAggregation(getChildTermsBuilder(groupFields,i+1,sumFields,avgFields,groupSize,groupSortMap));
}
}
/**
* 根据汇总键和SUM/AVG键,组合返回的查询值为MAP格式
* @param response
* @param groupFields
* @param sumFields
* @param avgFields
* @return
*/
private List<Map<String, Object>> getAggMap(SearchResponse response,List <String>groupFields,List<String> sumFields, List<String> avgFields){
List<Map<String, Object>> aggMaps = new ArrayList<Map<String, Object>>();
//首先获取最外层的AGG结果
Terms tempAggregation = response.getAggregations().get(groupFields.get(0));
//只有一个分组键不用进行递归
if(groupFields.size()==1){
for(Terms.Bucket tempBk:tempAggregation.getBuckets()){
Map<String, Object> tempMap = new HashMap<String, Object>();
tempMap.put(tempAggregation.getName(), tempBk.getKey());
for (Map.Entry<String, Aggregation> entry : tempBk.getAggregations().getAsMap().entrySet()) {
String key = entry.getKey();
if (sumFields.contains(key)) {
Sum aggSum = (Sum) entry.getValue();
double value = aggSum.getValue();
tempMap.put(key, value);
}
if (avgFields.contains(key)) {
Avg aggAvg = (Avg) entry.getValue();
double value = aggAvg.getValue();
tempMap.put(key, value);
}
}
aggMaps.add(tempMap);
}
}
else {
for (Terms.Bucket bk : tempAggregation.getBuckets()) {
//每个最外层的分组键生成一个键值对MAP
Map<String, Object> nkMap = new HashMap<String, Object>();
nkMap.put(tempAggregation.getName(), bk.getKey());
//通过递归的方式填充键值对MAP并加到最终的列表中
getChildAggMap(bk, 1, groupFields, sumFields, avgFields, nkMap, aggMaps);
}
}
return aggMaps;
}
/**
* 深层递归所有的AGG返回值,组合成最终的MAP列表
* @param bk 每次递归的单个Bucket
* @param i 控制分组键列表到了哪一层
* @param groupFields 分组键列表
* @param sumFields SUM键列表
* @param avgFields AVG键列表
* @param nkMap 键值对MAP
* @param aggMaps 最终结果的中间值
* @return
*/
private List<Map<String, Object>> getChildAggMap(Terms.Bucket bk,int i,List <String>groupFields,List<String> sumFields, List<String> avgFields,Map<String, Object> nkMap,List<Map<String, Object>> aggMaps){
if(i==groupFields.size()-1){
Terms tempAggregation = bk.getAggregations().get(groupFields.get(i));
for(Terms.Bucket tempBk:tempAggregation.getBuckets()){
Map<String, Object> tempMap = new HashMap<String, Object>();
tempMap.putAll(nkMap);
tempMap.put(tempAggregation.getName(), tempBk.getKey());
for (Map.Entry<String, Aggregation> entry : tempBk.getAggregations().getAsMap().entrySet()) {
String key = entry.getKey();
if (sumFields.contains(key)) {
Sum aggSum = (Sum) entry.getValue();
double value = aggSum.getValue();
tempMap.put(key, value);
}
if (avgFields.contains(key)) {
Avg aggAvg = (Avg) entry.getValue();
double value = aggAvg.getValue();
tempMap.put(key, value);
}
}
aggMaps.add(tempMap);
}
return aggMaps;
} else{
Terms tempAggregation = bk.getAggregations().get(groupFields.get(i));
for(Terms.Bucket tempBk:tempAggregation.getBuckets()){
nkMap.put(tempAggregation.getName(), tempBk.getKey());
getChildAggMap(tempBk, i + 1, groupFields, sumFields, avgFields, nkMap, aggMaps);
}
return aggMaps;
}
}
/**
* 将ES结果映射到指定对象
* @param resultMap
* @param cls
* @return
* @throws Exception
*/
public T mapResult(Map<String, Object> resultMap, Class<T> cls) throws Exception{
T result = cls.newInstance();
Field[] fields = cls.getDeclaredFields();
for (Field field : fields) {
Object object = resultMap.get(field.getName());
if (object != null) {
//根据几种基本类型做转换
if (field.getType().equals(Long.class) || field.getType().equals(long.class)) {
if(object.toString().indexOf('.')>0){
PropertyUtils.setProperty(result, field.getName(), Long.parseLong(object.toString().substring(0, object.toString().indexOf('.'))));
}else{
PropertyUtils.setProperty(result, field.getName(), Long.parseLong(object.toString()));
}
}else if (field.getType().equals(long.class) || field.getType().equals(long.class)) {
if(object.toString().indexOf('.')>0){
PropertyUtils.setProperty(result, field.getName(), Long.parseLong(object.toString().substring(0, object.toString().indexOf('.'))));
}else{
PropertyUtils.setProperty(result, field.getName(), Long.parseLong(object.toString()));
}
}else if (field.getType().equals(Integer.class) || field.getType().equals(Integer.class)) {
if(object.toString().indexOf('.')>0){
PropertyUtils.setProperty(result, field.getName(), Integer.parseInt(object.toString().substring(0, object.toString().indexOf('.'))));
}else{
PropertyUtils.setProperty(result, field.getName(), Integer.parseInt(object.toString()));
}
}else if (field.getType().equals(int.class) || field.getType().equals(int.class)) {
if(object.toString().indexOf('.')>0){
PropertyUtils.setProperty(result, field.getName(), Integer.parseInt(object.toString().substring(0, object.toString().indexOf('.'))));
}else{
PropertyUtils.setProperty(result, field.getName(), Integer.parseInt(object.toString()));
}
}else if (field.getType().equals(BigDecimal.class) || field.getType().equals(BigDecimal.class)) {
PropertyUtils.setProperty(result, field.getName(), BigDecimal.valueOf(Double.parseDouble(object.toString())));
}else if (field.getType().equals(Double.class) || field.getType().equals(Double.class)) {
PropertyUtils.setProperty(result, field.getName(), Double.parseDouble(object.toString()));
}else if (field.getType().equals(double.class) || field.getType().equals(double.class)) {
PropertyUtils.setProperty(result, field.getName(), Double.parseDouble(object.toString()));
}else if (field.getType().equals(Date.class) || field.getType().equals(Date.class)) {
PropertyUtils.setProperty(result, field.getName(), DateUtil.createDate(object.toString()));
}else if (field.getType().equals(String.class) || field.getType().equals(String.class)) {
PropertyUtils.setProperty(result, field.getName(), object);
}
}
}
return result;
}
/**
* 验证ES查询对象
* @param esQueryObj
* @throws Exception
*/
private void validationEsQuery(EsQueryObj esQueryObj) throws Exception{
if(StringUtils.isEmpty(esQueryObj.getIndexName())&&StringUtils.isEmpty(esQueryObj.getTypeName())){
throw new Exception("please check indexName and typeName");
}
}
/**
* 验证ES查询分组对象
* @param esQueryObj
* @throws Exception
*/
private void validationEsGroupQuery(EsQueryObj esQueryObj) throws Exception{
if(StringUtils.isEmpty(esQueryObj.getIndexName())&&StringUtils.isEmpty(esQueryObj.getTypeName())){
throw new Exception("please check indexName and typeName");
}
boolean groupOrderStatus=true;
st:for (Map.Entry<String, SortOrder> entry : esQueryObj.getGroupSortField().entrySet()) {
if(!esQueryObj.getSumFields().contains(entry.getKey())&&!esQueryObj.getAvgFields().contains(entry.getKey())){
groupOrderStatus=false;
break st;
}
}
if(!groupOrderStatus){
throw new Exception("please check groupSortField");
}
if (esQueryObj.getGroupFields().isEmpty() || esQueryObj.getGroupFields().size() <= 0 ||(esQueryObj.getSumFields().isEmpty()&&esQueryObj.getAvgFields().isEmpty())) {
throw new Exception("please check groupFields and sumFields and avgFields");
}
}
public EsClientFactory getEsClientFactory() {
return esClientFactory;
}
public void setEsClientFactory(EsClientFactory esClientFactory) {
this.esClientFactory = esClientFactory;
}
}
5. [代码]查询对象
import org.elasticsearch.index.query.AndFilterBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.search.sort.SortOrder;
import java.util.List;
import java.util.Map;
public class EsQueryObj {
/**
* ES索引名
*/
String indexName;
/**
* ES TYPE名(表名)
*/
String typeName;
/**
* 查询条件组合,类似于 "boolQuery().must( QueryBuilders.termQuery("opTime", "2016-03-30")).must(QueryBuilders.termQuery("operErpID", "xingkai"))"
*/
BoolQueryBuilder bq;
/**
* 查询条件组合,类似于 "boolQuery().must( QueryBuilders.termQuery("opTime", "2016-03-30")).must(QueryBuilders.termQuery("operErpID", "xingkai"))"
*/
AndFilterBuilder andFilterBuilder;
/**
* 分组键值列表,类似于group by 之后的字段
*/
List <String> groupFields;
/**
* 分组SUM键值列表
*/
List <String> sumFields;
/**
* 分组AVG键值列表
*/
List <String> avgFields;
/**
* 排序字段键值对,可以添加多个,类似于("opTime","DESC")
*/
Map<String,SortOrder> sortField;
/**
* 分组后排序字段键值对,可以添加多个,类似于("opTime","DESC"),注意此处最后PUT的键最先排序,排序键必须在sumFields或avgFields(只针对最后一层分组)
*/
Map<String,SortOrder> groupSortField;
/**
* 分组后返回数据数量,默认为100,最大不能超过500(只针对最后一层分组)
*/
int groupSize;
/**
* 取值的起始位置,默认为0
*/
int fromIndex;
/**
* 返回数据数量,默认为100,最大不能超过100000
*/
int size;
public String getIndexName() {
return indexName;
}
public void setIndexName(String indexName) {
this.indexName = indexName;
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public BoolQueryBuilder getBq() {
return bq;
}
public void setBq(BoolQueryBuilder bq) {
this.bq = bq;
}
public AndFilterBuilder getAndFilterBuilder() {
return andFilterBuilder;
}
public void setAndFilterBuilder(AndFilterBuilder andFilterBuilder) {
this.andFilterBuilder = andFilterBuilder;
}
public List<String> getGroupFields() {
return groupFields;
}
public void setGroupFields(List<String> groupFields) {
this.groupFields = groupFields;
}
public List<String> getSumFields() {
return sumFields;
}
public void setSumFields(List<String> sumFields) {
this.sumFields = sumFields;
}
public List<String> getAvgFields() {
return avgFields;
}
public void setAvgFields(List<String> avgFields) {
this.avgFields = avgFields;
}
public Map<String, SortOrder> getSortField() {
return sortField;
}
public void setSortField(Map<String, SortOrder> sortField) {
this.sortField = sortField;
}
public int getFromIndex() {
return fromIndex;
}
public void setFromIndex(int fromIndex) {
this.fromIndex = fromIndex;
}
public int getSize() {
return size;
}
public void setSize(int size) {
this.size = size;
}
public Map<String, SortOrder> getGroupSortField() {
return groupSortField;
}
public void setGroupSortField(Map<String, SortOrder> groupSortField) {
this.groupSortField = groupSortField;
}
public int getGroupSize() {
return groupSize;
}
public void setGroupSize(int groupSize) {
this.groupSize = groupSize;
}
}