elasticsearch的通用查询方法

最新推荐文章于 2022-08-10 21:30:00 发布
最新推荐文章于 2022-08-10 21:30:00 发布
阅读量1.8k 收藏 4
点赞数 1
分类专栏： Java 分布式开发
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/xiaopeng9275/article/details/82494142
版权

当数据量达到千万条，mysql数据库不能查询和插入的情况下，使用es来替代mysql，对dao层代码进行改造。essql是需要付费的，本人写了2个通用的查询方法，一个针对聚合查询，一个针对非聚合查询。

针对聚合查询和非聚合查询，有如下几个实现类：

  • ESCondition(封装查询条件)

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.elasticsearch.search.sort.SortOrder;

public class ESCondition {
	String index;
	String type;
	List<RangeParams> rangeList = new ArrayList<RangeParams>();//范围查询
	String searchFields;//指定查询哪些字段（用户非聚合查询）
	String countFields; //聚合统计
	String sumFields;//聚合求和
	String maxFields;
	String minFields;
	String avgFields;
	String matchStr; //精确查询
	String wildcardStr;//模糊查询
	String nullStr;//查询条件里面某个字段为空并且不为空串筛选
	String notNullStr;//查询条件里面某个字段不为空
	Map<String,List<String>> inField=new HashMap<String, List<String>>();//in查询
	String groupFields; 
	List<SortParams> sortList = new ArrayList<SortParams>();
	Integer size=10000;//默认取出10000条
	
	Integer from;
	//需要进行正则匹配的字段
	List<RegexParams> regexList = new ArrayList<RegexParams>();
	
	//格式  去重字段1，去重字段2
	String  distinctFields;
	
	
	
	
	public Integer getFrom() {
		return from;
	}
	public void setFrom(Integer from) {
		this.from = from;
	}
	public String getDistinctFields() {
		return distinctFields;
	}
	public void setDistinctFields(String distinctFields) {
		this.distinctFields = distinctFields;
	}
	public String getIndex() {
		return index;
	}
	public void setIndex(String index) {
		this.index = index;
	}
	public String getType() {
		return type;
	}
	public void setType(String type) {
		this.type = type;
	}
	
	
	
	public List<RegexParams> getRegexList() {
		return regexList;
	}
	public void addRegexList(String regexField,String regex) {
		RegexParams rp = new RegexParams(regexField,regex);
		regexList.add(rp);
	}
	
	public List<RangeParams> getRangeList() {
		return rangeList;
	}
	public void addRangeList(String rangeField,Object start,Object end) {
		RangeParams rp = new RangeParams(rangeField,start,end);
		rangeList.add(rp);
	}
	public void addRangeList(String rangeField,Object start,boolean includeLower,Object end,boolean includeUpper) {
		RangeParams rp = new RangeParams(rangeField,start,includeLower,end,includeUpper);
		rangeList.add(rp);
	}
	public String getSearchFields() {
		return searchFields;
	}
	public void setSearchFields(String searchFields) {
		this.searchFields = searchFields;
	}
	public String getCountFields() {
		return countFields;
	}
	public void setCountFields(String countFields) {
		this.countFields = countFields;
	}
	public String getSumFields() {
		return sumFields;
	}
	public void setSumFields(String sumFields) {
		this.sumFields = sumFields;
	}
	public String getMaxFields() {
		return maxFields;
	}
	public void setMaxFields(String maxFields) {
		this.maxFields = maxFields;
	}
	public String getMinFields() {
		return minFields;
	}
	public void setMinFields(String minFields) {
		this.minFields = minFields;
	}
	public String getAvgFields() {
		return avgFields;
	}
	public void setAvgFields(String avgFields) {
		this.avgFields = avgFields;
	}
	public String getMatchStr() {
		return matchStr;
	}
	public void setMatchStr(String matchStr) {
		this.matchStr = matchStr;
	}
	public String getGroupFields() {
		return groupFields;
	}
	public void setGroupFields(String groupFields) {
		this.groupFields = groupFields;
	}
	
	public List<SortParams> getSortList() {
		return sortList;
	}
	public void addSortFields(String field, SortOrder so) {
		SortParams sp = new SortParams(field,so);
		sortList.add(sp);
	}
	
	public Integer getSize() {
		return size;
	}
	public void setSize(Integer size) {
		this.size = size;
	}
	public String getWildcardStr() {
		return wildcardStr;
	}
	public void setWildcardStr(String wildcardStr) {
		this.wildcardStr = wildcardStr;
	}
	public Map<String, List<String>> getInField() {
		return inField;
	}
	public void addInField(String field,String inValues) {
		List<String> inList = Arrays.asList(inValues.split(","));
		inField.put(field, inList);
	}
	
	public void addInField(String field,List<String> inList) {
		inField.put(field, inList);
	}
	public String getNullStr() {
		return nullStr;
	}
	public void setNullStr(String nullStr) {
		this.nullStr = nullStr;
	}
	public String getNotNullStr() {
		return notNullStr;
	}
	public void setNotNullStr(String notNullStr) {
		this.notNullStr = notNullStr;
	}
    	
}

public class RangeParams {

	String rangeField;
	Object start;
	boolean includeLower=false;
	Object end;
	boolean includeUpper=false;
	String format;
	
	public RangeParams(String rangeField, Object start, Object end) {
		this.rangeField = rangeField;
		this.start = start;
		this.end = end;
	}
	
	public RangeParams(String rangeField, Object start, boolean includeLower, Object end, boolean includeUpper) {
		this.rangeField = rangeField;
		this.start = start;
		this.includeLower = includeLower;
		this.end = end;
		this.includeUpper = includeUpper;
	}
	public String getRangeField() {
		return rangeField;
	}
	public void setRangeField(String rangeField) {
		this.rangeField = rangeField;
	}
	public Object getStart() {
		return start;
	}
	public void setStart(Object start) {
		this.start = start;
	}
	public Object getEnd() {
		return end;
	}
	public void setEnd(Object end) {
		this.end = end;
	}
	public String getFormat() {
		return format;
	}
	public void setFormat(String format) {
		this.format = format;
	}
	public boolean isIncludeLower() {
		return includeLower;
	}
	public void setIncludeLower(boolean includeLower) {
		this.includeLower = includeLower;
	}
	public boolean isIncludeUpper() {
		return includeUpper;
	}
	public void setIncludeUpper(boolean includeUpper) {
		this.includeUpper = includeUpper;
	}
	
}

public class RegexParams {

	String regexfiled;
	
	String regex;

	public RegexParams(String regexfiled, String regex) {
		this.regexfiled = regexfiled;
		this.regex = regex;
	}

	public String getRegexfiled() {
		return regexfiled;
	}

	public void setRegexfiled(String regexfiled) {
		this.regexfiled = regexfiled;
	}

	public String getRegex() {
		return regex;
	}

	public void setRegex(String regex) {
		this.regex = regex;
	}
	
}

import org.elasticsearch.search.sort.SortOrder;

public class SortParams {
	String sortField;
	SortOrder sortOrder;
	public SortParams(String field, SortOrder so) {
		this.sortField = field;
		this.sortOrder = so;
	}
	public String getSortField() {
		return sortField;
	}
	public void setSortField(String sortField) {
		this.sortField = sortField;
	}
	public SortOrder getSortOrder() {
		return sortOrder;
	}
	public void setSortOrder(SortOrder sortOrder) {
		this.sortOrder = sortOrder;
	}
	
}
  • GroupBy（组合分组条件，聚合语句）
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;

public class GroupBy {


        private AggregationBuilder termsBuilder;
        
        //记录上次的子builder
        private AggregationBuilder lastGroupBuilder;

        public void addSubAgg(String aggName, String fieldName) {
            if(termsBuilder == null) {
                termsBuilder = AggregationBuilders.terms(aggName).field(fieldName).size(10000);
                lastGroupBuilder = termsBuilder;
            } else {
                AggregationBuilder n= AggregationBuilders.terms(aggName).field(fieldName).size(10000);
                lastGroupBuilder.subAggregation(n);
                lastGroupBuilder=n;
            }
        }
        
//        public void add
        
        public void addSumAgg(String aggName, String fieldName) {
            SumAggregationBuilder builder = AggregationBuilders.sum(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }


        public void addCountAgg(String aggName, String fieldName) {
            ValueCountAggregationBuilder builder = AggregationBuilders.count(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }


        public void addAvgAgg(String aggName, String fieldName) {
            AvgAggregationBuilder builder = AggregationBuilders.avg(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }


        public void addMinAgg(String aggName, String fieldName) {
            MinAggregationBuilder builder = AggregationBuilders.min(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }


        public void addMaxAgg(String aggName, String fieldName) {
            MaxAggregationBuilder builder = AggregationBuilders.max(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }

        public void addStatsAgg(String aggName, String fieldName) {
            StatsAggregationBuilder builder = AggregationBuilders.stats(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }

        public void addCardinalityAgg(String aggName, String fieldName) {
            CardinalityAggregationBuilder builder = AggregationBuilders.cardinality(aggName).field(fieldName);
            lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
        }

        //取出非聚合查询的字段，与聚合查询的字段合并
        public AggregationBuilder getTermsBuilder() {
            AggregationBuilder top = AggregationBuilders.topHits("top").explain(true);
//                    .size(3).from(0);
            lastGroupBuilder = lastGroupBuilder.subAggregation(top);
            return termsBuilder;
        }


        public void setTermsBuilder(AggregationBuilder termsBuilder) {
            this.termsBuilder = termsBuilder;
        }

        
}
  • AggUtil（用于聚合查询，不适用于非聚合查询）
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

import com.alibaba.fastjson.JSONObject;


public class AggUtil {


	private static Client client = null;
	
	
	/**
     *需要拿到一个客户端，在这里我们用的自己公司开发的客户端连接池。
     *
     *
     */   
	public static List<JSONObject>  getAggDocs(ESCondition c) {
    	
    	
    	ElasticHelper helper = ElasticClientPool.getInstance();
    	List<JSONObject> results=null;
    	GroupBy gb = new GroupBy();
    	List<RangeParams> rangeList = c.getRangeList();
    	List<SortParams> sortList = c.getSortList();
    	List<RegexParams> regexList = c.getRegexList();
    	Map<String,List<String>> inMap = c.getInField();
    	String groupArr[] = null,countArr[] = null,maxArr[] = null ,minArr[] = null ,sumArr[] = null ,avgArr[] = null ;
    	try {             
		    	TransportClient client = helper.getClient();
		        SearchRequestBuilder searchRequest = client.prepareSearch(c.getIndex());
		        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
		        if(rangeList != null && rangeList.size() != 0) {
		        	for(int i=0 ; i < rangeList.size() ; i++) {
		        		RangeParams rp = rangeList.get(i);
		        		boolQuery.must(
		        				QueryBuilders.rangeQuery(rp.getRangeField())
		        				.from(rp.getStart())
		        				.to(rp.getEnd())
		        				.includeLower(true)
		        				.includeUpper(true));
		        	}
		        }
		        if(regexList != null && regexList.size() != 0) {
		        	for(int i=0 ; i < regexList.size() ; i++) {
		        		RegexParams rp = regexList.get(i);
		        		boolQuery.must(
		        				QueryBuilders.regexpQuery(rp.getRegexfiled(),rp.getRegex()));
		        	}
		        }
		        if (StringUtils.isNotEmpty(c.getMatchStr())) {
		            for (String s : c.getMatchStr().split(",")) {
		                String[] ss = s.split("=");
		                if(ss.length > 1){
		                    boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1]));
		                }
		            }
		        }
		        //in查询
		        if (inMap.size() != 0) {
		     	   for(Map.Entry<String, List<String>> entry : inMap.entrySet() ) {
		     		   boolQuery.must(QueryBuilders.termsQuery(entry.getKey(), entry.getValue()));
		     	   }
		        }
		        searchRequest.setQuery(boolQuery);
		        
		        
		        //查询哪些字段
		        if (StringUtils.isNotEmpty(c.getSearchFields())) {
		            searchRequest.setFetchSource(c.getSearchFields().split(","), null);
		            searchRequest.setFetchSource(true);
		        }
		        //查询分组
		        if (StringUtils.isNotEmpty(c.getGroupFields())) {
		        	groupArr = c.getGroupFields().split(",");
		        	
		        	for(String kvField : groupArr) {
				        String kv[] = kvField.split(":"); 
		        		gb.addSubAgg(kv[0], kv[1]);
		        	}
		        }
		        //count统计
		        if (StringUtils.isNotEmpty(c.getCountFields())) {
			        countArr = c.getCountFields().split(",");
			        for(String kvField : countArr) {
			        	String kv[] = kvField.split(":"); 
			        	gb.addCountAgg(kv[0], kv[1]);
			        }
		        }
		        //sum统计
		        if (StringUtils.isNotEmpty(c.getSumFields())) {
			        sumArr = c.getSumFields().split(",");
			        for(String kvField : sumArr) {
			        	String kv[] = kvField.split(":");
			        	gb.addSumAgg(kv[0], kv[1]);
			        }
		        }
		        //max统计
		        if (StringUtils.isNotEmpty(c.getMaxFields())) {
			        maxArr = c.getMaxFields().split(",");
			        for(String kvField : maxArr) {
			        	String kv[] = kvField.split(":");
			        	gb.addMaxAgg(kv[0], kv[1]);
			        }
		        }
		        //min统计
		        if (StringUtils.isNotEmpty(c.getMinFields())) {
			        minArr = c.getMinFields().split(",");
			        for(String kvField : minArr) {
			        	String kv[] = kvField.split(":");
			        	gb.addMinAgg(kv[0], kv[1]);
			        }
		        }
		        //avg统计
		        if (StringUtils.isNotEmpty(c.getAvgFields())) {
			        avgArr = c.getAvgFields().split(",");
			        for(String kvField : avgArr) {
			        	String kv[] = kvField.split(":");
			        	gb.addAvgAgg(kv[0], kv[1]);
			        }
		        }
		        
		        if (sortList != null && sortList.size()!=0) {
		     	   for(int i=0 ; i < sortList.size() ; i++) {
		     		   SortParams sp = sortList.get(i);
		     		   searchRequest.addSort(sp.getSortField(), sp.getSortOrder());
		     	   }
		        }
	            if(c.getSize() != null) {
	            	searchRequest.setSize(c.getSize());
	            }
	            SearchResponse response = null;
	            
	            searchRequest.addAggregation(gb.getTermsBuilder());
	            
	            response= searchRequest.setSearchType(SearchType.QUERY_THEN_FETCH).execute().actionGet();
	            Map<String,Aggregation> result = response.getAggregations().asMap();
	            results = groupTree(groupArr,0,result,countArr,maxArr,minArr,sumArr,avgArr);
        } catch (Exception e) {
        	e.printStackTrace();
           logger.error(e.getMessage(),e);
        }finally {
        	ElasticClientPool.freeElasticClient(helper);
		}
        return results;
    }

	private static List<JSONObject> groupTree(String[] groupArr,int index,Map<String, Aggregation> result, String[] countArr, String[] maxArr, String[] minArr, String[] sumArr, String[] avgArr) {
		List<JSONObject> jsonList = new ArrayList<JSONObject>();
		if(index==groupArr.length-1) {
			 Terms terms = (Terms) /*(StringTerms)*/ result.get(groupArr[index].split(":")[0]);
			 for(Terms.Bucket bt : terms.getBuckets()) {
				 //先取出基础字段
				 TopHits topHits = bt.getAggregations().get("top");
				 //聚合只需取每组第一条封装
	         	 SearchHit first = topHits.getHits().getHits()[0];
	         	 JSONObject jsonObject = JSONObject.parseObject(first.getSourceAsString());
	         	 
	         	jsonObject.put("docCount", bt.getDocCount());
	         	 if(countArr != null){
					 for(String countField : countArr) {
						 String kv[]=countField.split(":");
						 ValueCount count = bt.getAggregations().get(kv[0]);
						 jsonObject.put(kv[0], count.getValue());
					 }
				 }
	         	 if(maxArr != null) {
	         		 for(String maxField : maxArr) {
	         			 String kv[]=maxField.split(":");
						 Max max = bt.getAggregations().get(kv[0]);
						 jsonObject.put(kv[0], max.getValue());
					 } 
	         	 }
				 if(minArr != null) {
					 for(String minField : minArr) {
						 String kv[]=minField.split(":");
						 Min min = bt.getAggregations().get(kv[0]);
						 jsonObject.put(kv[0], min.getValue());
					 }
				 }
				 if(sumArr != null) {
					 for(String sumField : sumArr) {
						 String kv[]=sumField.split(":");
						 Sum sum = bt.getAggregations().get(kv[0]);
						 jsonObject.put(kv[0], sum.getValue());
					 }
				 }
				 if(avgArr != null) {
					 for(String avgField : avgArr) {
						 String kv[]=avgField.split(":");
						 Avg avg = bt.getAggregations().get(kv[0]);
						 jsonObject.put(kv[0], avg.getValue());
					 }
				 }
				 jsonList.add(jsonObject);
         	}
		} else {
			Terms terms = (Terms) result.get(groupArr[index].split(":")[0]);
			if(terms != null) {
				for(Terms.Bucket bt : terms.getBuckets()) {
					Map<String, Aggregation> result2 = bt.getAggregations().asMap();
					List<JSONObject> jsonList2 = groupTree(groupArr, index+1,  result2, countArr, maxArr, minArr, sumArr, avgArr);
					jsonList.addAll(jsonList2);
				}
			} else {
				return new ArrayList<JSONObject>();
			}
		}
		return jsonList;
	}

}
  • ListUtil（用于非聚合查询）


import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.search.MultiMatchQuery.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms.Bucket;
import org.elasticsearch.search.sort.SortOrder;

import com.alibaba.fastjson.JSONObject;

 
 
 
 
public class ListUtil {
   
   public static List<JSONObject> getListDocs(ESCondition c) {
	   ElasticHelper helper = ElasticClientPool.getInstance();
	   TransportClient client = helper.getClient();
       BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
       SearchRequestBuilder searchRequest = client.prepareSearch(c.getIndex());
       List<RangeParams> rangeList = c.getRangeList();
       List<SortParams> sortList = c.getSortList();
       List<RegexParams> regexList = c.getRegexList();
       Map<String,List<String>> inMap = c.getInField();
       if(rangeList != null && rangeList.size() != 0) {
    	   for(int i=0 ; i < rangeList.size() ; i++) {
    		   RangeParams rp = rangeList.get(i);
    		   boolQuery.must(
               QueryBuilders.rangeQuery(rp.getRangeField())
                       .from(rp.getStart())
                       .to(rp.getEnd())
                       .includeLower(rp.isIncludeLower())
                       .includeUpper(rp.isIncludeUpper()));
    	   }
       }
       //field is null 
       if (StringUtils.isNotEmpty(c.getNullStr())) {
           for (String field : c.getNullStr().split(",")) {
        	   boolQuery.must(QueryBuilders.existsQuery(field)).mustNot(QueryBuilders.regexpQuery(field, ".+"));
           }
       }
       
       if (StringUtils.isNotEmpty(c.getNotNullStr())) {
           for (String field : c.getNotNullStr().split(",")) {
        	   boolQuery.must(QueryBuilders.existsQuery(field)).must(QueryBuilders.regexpQuery(field, ".+"));
           }
       }
       
       if(regexList != null && regexList.size() != 0) {
       	for(int i=0 ; i < regexList.size() ; i++) {
       		RegexParams rp = regexList.get(i);
       		boolQuery.must(
       				QueryBuilders.regexpQuery(rp.getRegexfiled(),rp.getRegex()));
       	}
       }
       if (StringUtils.isNotEmpty(c.getMatchStr())) {
           for (String s : c.getMatchStr().split(",")) {
               String[] ss = s.split(":");
               if(ss.length > 1){
                   boolQuery.must(QueryBuilders.matchQuery(ss[0], ss[1]));
               }
           }
       }
       if (inMap.size() != 0) {
    	   for(Map.Entry<String, List<String>> entry : inMap.entrySet() ) {
    		   boolQuery.must(QueryBuilders.termsQuery(entry.getKey(), entry.getValue()));
    	   }
       }
       //模糊查询
       if (StringUtils.isNotEmpty(c.getWildcardStr())) {
    	   BoolQueryBuilder or = QueryBuilders.boolQuery();
           for (String s : c.getWildcardStr().split(",")) {
               String[] ss = s.split(":");
               if(ss.length > 1){
            	   or = or.should(QueryBuilders.wildcardQuery(ss[0], "*"+ss[1]+"*"));
               }
           }
           boolQuery.must(or);
       }
       searchRequest.setQuery(boolQuery);
       if (StringUtils.isNotEmpty(c.getSearchFields())) {
           searchRequest.setFetchSource(c.getSearchFields().split(","), null);
       }
       if (sortList != null && sortList.size()!=0) {
    	   for(int i=0 ; i < sortList.size() ; i++) {
    		   SortParams sp = sortList.get(i);
    		   searchRequest.addSort(sp.getSortField(), sp.getSortOrder());
    	   }
       }
       searchRequest.setFetchSource(true);
       
       if(c.getFrom()==null &&c.getSize()!=null){
    	   searchRequest.setSize(c.getSize());
       }else if (c.getFrom()!=null &&c.getSize()!=null){
    	   searchRequest.setFrom(c.getFrom()).setSize(c.getSize());
       }
       
       List<JSONObject> sourceList = null;

       try {
           SearchResponse response = searchRequest
                   .execute()
                   .actionGet();
           SearchHits shs = response.getHits();
           sourceList = new ArrayList<JSONObject>();
           for (SearchHit sh : shs) {
        	   JSONObject jsonObject = JSONObject.parseObject(sh.getSourceAsString());
        	   jsonObject.put("id", sh.getId());
               sourceList.add(jsonObject);
           }
       } catch (Exception e) {
           e.printStackTrace();
       }
       return sourceList;
   }
	   
}

ES数据源改造就这些，能对付大部分sql查询，除了一些三元表达式，特殊函数。唯一的不足就是一次只能查最大10000条，这个要根据自己的数据量，在建表的时候设置上限。查询出来的结果返回的是一个json数据格式的字符串列表，自己再逐个取出需要的字段回填到对象就OK了

耕耘的小象
关注
  • 1
    点赞
  • 踩
  • 4
    收藏
    觉得还不错? 一键收藏
  • 2
    评论
专栏目录
elasql:通用SQL到Elasticsearch DSL查询转换器
04-28
通用SQL到Elasticsearch DSL查询转换器。 专为设计 地位 实验性-加入我们，骇客入侵！ 安装 npm install elasql 用法 const convert = require('elasql').convert convert('SELECT id,name FROM shop WHERE shop_id...
Go实战全家桶之八：统一ES服务接口之通用查询嵌套查询之封装与增删改API
最新发布
leijmdas的专栏
05-24 1202
"msg": "成功",
ElasticSearch 高级查询语法
星空下的博客
06-27 1331
ElasticSearch 高级查询语法Query DSL
枚举中找不到常量 几种解决方案
qq_35490522的博客
07-29 1312
Caused by: java.lang.IllegalArgumentException: No enum constant 1.可能真的没有常量的问题（把报错所需常量添加到枚举常量就可以） 2.不是接口注入的方式，是以 构造注入的方式，要编写无参和有参的构造函数 3.setter注入，要检查属性是否有setter 4.注解注入，要检查注解是否有写 ...
ElasticSearch聚合和OR条件查询
热门推荐
奔跑的蜗牛
02-14 2万+
在使用ElasticSearch中经常会遇到统计、查询需求，实现类似sql分组计算、条件查询的语法，ES在这些方面都支持的较不错，用起来也比较方便。笔者就自己开发中用到的Java API举例如下。 分组聚合查询 实现类似select avg(executionTime) from t WHERE executionDate between 'A' AND 'B' group by execut...
ElasticSearch如何使用以及java代码如何查询并排序ES中的数据（距离排序）
m0_37914467的博客
09-10 6772
1.环境准备 elasticsearch压缩包、kibana插件、head-master插件，具体怎么安装配置，小伙伴们可以自行百度网上很多案例 可参考：https://blog.csdn.net/sinat_42338962/article/details/85227902 这里说一下，国内下载一般都很慢，可以参考这篇文章，博主都放在百度云，下载很方便 文章链接：https://blog.csdn.net/weixin_37281289/article/details/101483434 2.h
es中java代码高亮显示_Elasticsearch（七）使用java代码实现es 分页 高亮 排序 查询功能...
weixin_42386361的博客
02-23 709
在es api中我们可以看到，其查询条件主要封在QueryBuilder中；分页、排序、字段过滤、高亮主要封装在SourceBuilder中；测试代码如下：package com.xiaohui;import org.apache.http.HttpHost;import org.elasticsearch.action.search.MultiSearchRequest;import org.e...
C# ES通用类demo
03-01
本示例"C# ES通用类demo"专注于如何在C#环境中与Elasticsearch进行交互，实现数据的增删改查操作。下面我们将深入探讨这些知识点。 1. **C#基础**：C#是.NET框架的主要编程语言，由微软开发，支持面向对象编程，...
elasticsearch5.6以上version通用java API
03-18
在"elasticsearch5.6以上version通用java API"中，我们将会探讨如何利用Java编写工具类，以覆盖更全面的操作，如映射创建、批量插入、聚合查询以及模糊和精确查询。 1. **映射创建**：Elasticsearch的映射（Mapping...
elastic-common:Elasticsearch通用模块
05-01
Elasticsearch通用模块 设置项目 从安装Gradle版本4.5.1 Git克隆此项目git clone https://github.com/polyglotted/elastic-common并插入cd 运行ESDocker docker容器docker docker run -d --name esdocker -p 9200:...
es2csv：从Elasticsearch导出到CSV文件
01-30
**es2csv：从Elasticsearch导出到CSV文件** `es2csv`是一个非常实用的工具，主要用于将数据从Elasticsearch索引导出到CSV格式的文件中。这个工具通常由Python编写，适用于那些需要对存储在Elasticsearch中的大量...
elasticsearch6聚合排序总结 - JavaAPI
tiansheng1225的博客
09-12 9169
序言:         近来一直在javaAPI对ES数据进行统计分析, 中间也遇到不少问题, 本篇文章只针对聚合过程中的排序方法进行总结 例如:select  avg(age) as avg_age from table group by gender;   聚合的两种排序方式        1. 按组内字段排序: 分组后按组内数据某个字段进行排序        2. 按聚合后指标排...
Elasticsearch之排序解析(十二)
ntzzzsj的博客
08-10 4030
在默认情况下，ES对搜索结果是按照相关性降序排序的。有时需要按照某些字段的值进行升序或者降序排序。ES提供了sort子句可以对数据进行排序。使用sort子句一般是按照字段信息进行排序，不受相关性影响，而且打分步骤需要耗费一定的硬件资源和时间，因此默认情况下，不对文档进行打分。使用sort排序分为两种类别，一种是按照字段值的大小进行排序，另一种是按照给定地理坐标的距离远近进行排序。...
es实现多字段聚合后再分页-附JAVA工具类
u011250186的博客
07-26 2806
es实现多字段聚合后再分页-附JAVA工具类
elasticsearch 索引_Elasticsearch 中的索引排序
weixin_39825322的博客
11-20 529
在Elasticsearch6.0中，引入了一个新的特征，叫Index Sorting（索引排序）。用户可以将索引数据按照指定的顺序存储在硬盘上，这样在搜索排序取前N条时，不需要访问所有的匹配中的记录再进行排序，只需要访问前N条记录即可。Lucene中的Index Sorting很多年以前，Lucene引入了一个工具叫做IndexSorter。这个工具的作用是从一个已有的索引中取数，然后按照指定的...
es自定义排序
weixin_32128491的博客
09-21 3989
## 排序 ### 一、默认排序规则 默认情况下，是按照_score降序排序。 _score使用的算法,计算出一个索引中的文本，与搜索文本，他们之间的关联匹配程度 es使用的是，term frequency和inverse documnet frequency算法，简称为TF/IDF算法 term frequency:搜索文本中的各个词条在field文本中出现了多少次，出现次数越多，分数越高 inverse documnet frequency：搜索文本中的各个词条在整个索引的所有文档中出现了多少次，出
Java实现elasticsearch 多字段分组汇聚及分页
非衣鲲化的博客
05-09 5606
由于elastic search自身没有提供汇聚以后进行分组的方法，所以需要自己通过代码来实现汇聚以后的分组功能。 下面是代码： package com.qq.test; import com.qq.entity.User; import com.sun.javafx.binding.SelectBinding; import com.sun.org.apache.bcel.inte...
ElasticSearch：Aggregation聚合查询的入门与进阶
u012998680的博客
08-31 7606
目录 桶(Buckets) 指标(Metrics) 将两者结合起来——聚合 常见的聚合查询 聚合查询的使用 1、简单的词频统计 2、数据按时间划分 3、数据按某个字段进行聚合后，再按时间排序 为了掌握聚合，要先了解两个主要概念： Buckets(桶)：满足某个条件的文档集合。 Metrics(指标)：为某个桶中的文档计算得到的统计信息。 就是这样！每个聚合只是简单地由一个或者多个桶，零个或者多个指标组合而成。可以将它粗略地转换为SQL： SELECT COUNT(field) F
elasticsearch通用工具类
05-25
以下是一个基本的 Elasticsearch 工具类，可以用于与 Elasticsearch 交互： ```java import org.apache.http.HttpHost; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.List; import java.util.Map; public class ElasticsearchUtil { private RestHighLevelClient client; public ElasticsearchUtil(String host, int port) { client = new RestHighLevelClient(RestClient.builder(new HttpHost(host, port, "http"))); } public void close() throws IOException { client.close(); } public boolean indexExists(String index) throws IOException { return client.indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT); } public boolean createIndex(String index) throws IOException { CreateIndexRequest request = new CreateIndexRequest(index); request.settings(Settings.builder() .put("index.number_of_shards", 3) .put("index.number_of_replicas", 2) ); request.mapping("properties", "{\n" + " \"name\": {\n" + " \"type\": \"text\"\n" + " },\n" + " \"age\": {\n" + " \"type\": \"integer\"\n" + " }\n" + "}", XContentType.JSON); request.alias(new Alias("my_alias")); CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT); return response.isAcknowledged(); } public boolean deleteIndex(String index) throws IOException { DeleteIndexRequest request = new DeleteIndexRequest(index); AcknowledgedResponse response = client.indices().delete(request, RequestOptions.DEFAULT); return response.isAcknowledged(); } public boolean documentExists(String index, String id) throws IOException { GetRequest getRequest = new GetRequest(index, id); getRequest.fetchSourceContext(new FetchSourceContext(false)); getRequest.storedFields("_none_"); return client.exists(getRequest, RequestOptions.DEFAULT); } public boolean createDocument(String index, String id, Map<String, Object> source) throws IOException { IndexRequest request = new IndexRequest(index); if (id != null) { request.id(id); } request.source(source); IndexResponse response = client.index(request, RequestOptions.DEFAULT); return response.getResult() == DocWriteResponse.Result.CREATED; } public boolean updateDocument(String index, String id, Map<String, Object> source) throws IOException { UpdateRequest request = new UpdateRequest(index, id); request.doc(source); UpdateResponse response = client.update(request, RequestOptions.DEFAULT); return response.getResult() == DocWriteResponse.Result.UPDATED; } public boolean deleteDocument(String index, String id) throws IOException { DeleteRequest request = new DeleteRequest(index, id); DeleteResponse response = client.delete(request, RequestOptions.DEFAULT); return response.getResult() == DocWriteResponse.Result.DELETED; } public Map<String, Object> getDocument(String index, String id) throws IOException { GetRequest getRequest = new GetRequest(index, id); GetResponse response = client.get(getRequest, RequestOptions.DEFAULT); if (response.isExists()) { return response.getSource(); } else { return null; } } public List<Map<String, Object>> searchDocument(String index, String field, String value) throws IOException { QueryBuilder queryBuilder = QueryBuilders.matchQuery(field, value); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(queryBuilder); SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(sourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); List<Map<String, Object>> result = new ArrayList<>(); for (SearchHit hit : searchResponse.getHits().getHits()) { result.add(hit.getSourceAsMap()); } return result; } public boolean bulkCreateDocuments(String index, List<Map<String, Object>> sources) throws IOException { BulkRequest request = new BulkRequest(); for (Map<String, Object> source : sources) { IndexRequest indexRequest = new IndexRequest(index); indexRequest.source(source); request.add(indexRequest); } BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); return !response.hasFailures(); } public boolean bulkUpdateDocuments(String index, List<Map<String, Object>> sources) throws IOException { BulkRequest request = new BulkRequest(); for (Map<String, Object> source : sources) { String id = (String) source.remove("id"); UpdateRequest updateRequest = new UpdateRequest(index, id); updateRequest.doc(source); request.add(updateRequest); } BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); return !response.hasFailures(); } public boolean bulkDeleteDocuments(String index, List<String> ids) throws IOException { BulkRequest request = new BulkRequest(); for (String id : ids) { DeleteRequest deleteRequest = new DeleteRequest(index, id); request.add(deleteRequest); } BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); return !response.hasFailures(); } } ``` 这个工具类使用 Elasticsearch 的 Java High Level REST Client，支持创建、删除索引，增删改查文档，以及批量操作。

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 2
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值