项目目录
pom文件
<artifactId>paascloud-common-es</artifactId>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.5.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.5.1</version>
</dependency>配置
@Configuration
public class ESRestClientConfig {
private static int connectTimeOut = 5000; // 连接超时时间
private static int socketTimeOut = 90000; // 连接超时时间
private static int connectionRequestTimeOut = 1000; // 获取连接的超时时间
private static int maxConnectNum = 100; // 最大连接数
private static int maxConnectPerRoute = 100; // 最大路由连接数
@Value("${es.host}")
private String host ;
@Value("${es.port}")
private Integer port ;
@Bean("restHighLevelClient")
public RestHighLevelClient restHighLevelClient(){
RestClientBuilder builder = RestClient.builder(new HttpHost(host,port,"http"));
builder.setRequestConfigCallback(new RequestConfigCallback() {
@Override
public Builder customizeRequestConfig(Builder requestConfigBuilder) {
requestConfigBuilder.setConnectTimeout(connectTimeOut);
requestConfigBuilder.setSocketTimeout(socketTimeOut);
requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
return requestConfigBuilder;
}
});
// 异步httpclient连接数配置
builder.setHttpClientConfigCallback(new HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.setMaxConnTotal(maxConnectNum);
httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);
return httpClientBuilder;
}
});
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
//如果是集群
// RestHighLevelClient client = new RestHighLevelClient(
// RestClient.builder(new HttpHost("127.0.0.1", 9200, "http")
// , new HttpHost("127.0.0.1", 9201, "http")
// , new HttpHost("127.0.0.1", 9202, "http")));
// return client;
}
}model
@Data
public abstract class ESBaseVo implements Serializable {
private static final long serialVersionUID = 532876990890744407L;
@IndexType(type = IndexTypeConst.KEYWORD)
private String id;
}
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE})
public @interface EsClass {
String indexName() default "";
String type() default "_doc";
}
/**
* 索引字段类型
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.ANNOTATION_TYPE})
public @interface IndexSearchType {
//字段类型
String type() default "";//参考IndexSearchTypeConst
String column() default "";//对应es字段名称（当入参多个对应一个字段，或者 list对应到string 类型字段，则2个属性的column名称于es字段一致）
String sortType() default "" ;//desc,asc
}
/**
* ES index 字段类型枚举类
*/
public class IndexSearchTypeConst {
/**
* 检索类型:match（模糊匹配），term（精准匹配），id（匹配ID，多ID以逗号分隔），multiple（匹配多值，多值以逗号分隔）,multiples（匹配多值，多值以$分隔）,
* date(时间条件，date_from:大于时间,date_to：小于时间),
* gt(大于), lt(小于), lte(小于等于), gte(大于等于))
* and_include_%_or_%（SQL:and (x like %xxx% or y like %yyy%); java参数：xxx$yyy$zzz xxx、yyy、zzz以$隔开）
* exist(不为null),notExist(为null),
* sort排序,
*/
/**
* 检索类型:match（模糊匹配）
*/
public final static String match = "match";
public final static String should = "should";
/**
* term（精准匹配）
*/
public final static String term = "term";
/**
* id（匹配ID，多ID以逗号分隔）
*/
public final static String id = "id";
/**
* multiples，multiple（匹配多值）
*/
public final static String multiple = "multiple";
/**
* date(时间条件，date_from:大于时间,date_to：小于时间),
*/
public final static String date_from = "date_from";
/**
* date(时间条件，date_from:大于时间,date_to：小于时间),
*/
public final static String date_to = "date_to";
/**
* gt(大于), lt(小于), lte(小于等于), gte(大于等于))
*/
public final static String gt = "gt";
/**
* gt(大于), lt(小于), lte(小于等于), gte(大于等于))
*/
public final static String lt = "lt";
/**
* gt(大于), lt(小于), lte(小于等于), gte(大于等于))
*/
public final static String lte = "lte";
/**
* gt(大于), lt(小于), lte(小于等于), gte(大于等于))
*/
public final static String gte = "gte";
/**
* and_include_%_or_%（SQL:and (x like %xxx% or y like %yyy%); java参数：xxx$yyy$zzz xxx、yyy、zzz以$隔开）
*/
public final static String likeOr = "and_include_%_or_%";
/**
* //SQL:and x in(value1,value2 ) (or x.is inull or x=='' ));
*/
public final static String orIsNull = "or_is_null";
/**
* //SQL:and (x in(value1,value2) );
*/
public final static String orNotNull = "or_not_null";
/**
* exist(不为null)
*/
public final static String exist = "exist";
/**
* notExist(为null)
*/
public final static String notExist = "notExist";
/**
* //SQL: and xxx > yyy （字段xxx > 字段yyy）
*/
public final static String gtInWwoWords = "gt_in_two_words";
/**
* //SQL: and xxx <= yyy （字段xxx <= 字段yyy）
*/
public final static String lteInTwoWords = "lte_in_two_words";
public final static String commaStr = "commaStr";
/**
* es 脚本
*/
public final static String script = "script";
/**
* 排序
*/
public final static String sort = "sort";
}
/**
* 索引字段类型
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.ANNOTATION_TYPE})
public @interface IndexType {
//字段类型
String type() default "text";//参考IndexTypeConst
//分词器，中文字段，且需要全局匹配的才设置,ik_smart:粗粒度的分词。ik_max_word：最细粒度的分词
String analyzer() default "";
//查询时是否以指定分词规则来匹配,ik_smart
String searchAnalyzer() default "";
//日期格式化
String format() default "";
}/**
* ES index 字段类型枚举类
*/
public class IndexTypeConst {
public final static String INTEGER = "integer";//数值类型
public final static String DOUBLE = "double";//数值类型
public final static String TEXT = "text";//文本,会进行分词处理
public final static String KEYWORD = "keyword";//文本，不进行分词处理
public final static String DATE = "date";//日期
public final static String BOOLEAN = "boolean";//布尔值
public final static String LONG = "long";//数值类型
public final static String FLOAT = "float";//数值类型
}
tools包
@Slf4j
/**
* es工具类
*/
@Component
public class ESRestClientUtils {
private static final String esType="_doc";
private static final int defaultPageSize = 3000;
@Autowired
private RestHighLevelClient restHighLevelClient;
private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
/**
* 查询总数
* @param searchRequest
* @return
*/
public int scrollSearchAllCount(SearchRequest searchRequest) {
int total=0;
//设置游标有效期
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(3L));
searchRequest.scroll(scroll);
SearchResponse searchResponse = null;
try {
searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
Long totalHits = searchResponse.getHits().getTotalHits();
//释放游标
total=totalHits.intValue();
if (scrollId != null) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
}
} catch (IOException e) {
log.error("获取index 总数失败："+e.getMessage());
}
return total;
}
/**
* 查询es 分页
* @param currentPage
* @param pageSize
* @param searchRequest
* @return
*/
public SearchResponse scrollSearchAllByPage(int currentPage, int pageSize, SearchRequest searchRequest) {
int from = (currentPage - 1) * pageSize;
int size = pageSize;
//设置游标有效期
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(3L));
searchRequest.scroll(scroll);
SearchResponse searchResponse = null;
try {
searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
int maxNum = from / size;
for (int i = 0; i < maxNum; i++) {
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
}
//释放游标
if (scrollId != null) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
}
} catch (IOException e) {
log.error("获取index 分页失败："+e.getMessage());
}
return searchResponse;
}
public String saveIndex(String indexName,Class<?> clazz) {
return saveIndex(indexName,null,clazz);
}
/**
* 新增索引
* @param indexName
* @return
*/
public String saveIndex(String indexName,String typeName,Class<?> clazz) {
try {
PublicUtil.isParam(indexName,"索引名称不能为空");
if(existIndex(indexName)){
PublicUtil.isParam(indexName,"索引名称已存在");
return "fail";
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
createIndexRequest.settings(Settings.builder()
.put("index.number_of_shards", 5)
.put("index.number_of_replicas", 1)
);
//封装属性 类似于json格式
XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties");
for(; clazz != Object.class; clazz = clazz.getSuperclass()) {
for (Field field : clazz.getDeclaredFields()) {
if (!Modifier.isStatic(field.getModifiers())) {
IndexType indexType = field.getAnnotation(IndexType.class);
if(PublicUtil.isNotEmpty(indexType)){
builder.startObject(field.getName()).field("type", indexType.type());
if(IndexTypeConst.TEXT.equals(indexType.type())) {
builder.field("fielddata", "true");
}
String analyzer = indexType.analyzer();//分析器
if(analyzer != null && !"".equals(analyzer)){
builder.field("analyzer", analyzer);
}
String searchAnalyzer = indexType.searchAnalyzer();//语言解析器
if(searchAnalyzer != null && !"".equals(searchAnalyzer)){
builder.field("search_analyzer", searchAnalyzer);
}
String format = indexType.format();//时间格式
if(format != null && !"".equals(format)){
builder .field("format", format);
}
}else{
builder.field("type", "text");
builder.field("fielddata", true);
}
builder.field("index", true);
builder.endObject();
}
}
}
builder.endObject().endObject();
createIndexRequest.mapping(typeName==null?esType:typeName,builder);
restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
log.error("创建索引失败："+e.getMessage());
}
return "success";
}
/**
* 校验索引是否存在
* @param indexName
* @return
*/
public boolean existIndex(String indexName) {
PublicUtil.isParam(indexName,"索引名称不能为空");
try {
GetIndexRequest qyf = new GetIndexRequest();
qyf.indices(indexName);
return restHighLevelClient.indices().exists(qyf, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
public <T extends ESBaseVo> String insertIndexData(String indexName, ESBaseVo eSBaseVo) {
return insertIndexData(indexName,null,eSBaseVo);
}
/**
* 单个添加索引内容（根据id）
* @param indexName
* @return
*/
public <T extends ESBaseVo> String insertIndexData(String indexName, String typeName,ESBaseVo eSBaseVo) {
try {
PublicUtil.isParam(indexName,"索引名称不能为空");
PublicUtil.isParam(eSBaseVo,"对象不能为空");
PublicUtil.isParam(eSBaseVo.getId(),"id不能为空");
IndexRequest request = new IndexRequest(indexName);
request.source(JSON.toJSONString(eSBaseVo), XContentType.JSON);
request.type(typeName==null?esType:typeName);
request.id(eSBaseVo.getId());
IndexResponse index = restHighLevelClient.index(request, RequestOptions.DEFAULT);
return index.toString();
} catch (IOException e) {
log.error("添加索引内容失败: "+e.getMessage());
}
return "success";
}
/**
* 根据id 查询索引内容
* @param indexName
* @param id
* @return
*/
public String getIndexDataById(String indexName,String id) {
PublicUtil.isParam(indexName,"索引名称不能为空");
PublicUtil.isParam(id,"id不能为空");
GetResponse documentFields = null;
try {
GetRequest qyf = new GetRequest(indexName);
qyf.type(esType);
qyf.id(id);
documentFields = restHighLevelClient.get(qyf, RequestOptions.DEFAULT);
return documentFields.getSourceAsString();
} catch (IOException e) {
log.error("查询单个索引内容失败: "+e.getMessage());
}
return null;
}
public <T extends ESBaseVo> String updateIndexDataById(String indexName,ESBaseVo eSBaseVo) {
return updateIndexDataById(indexName,null,eSBaseVo);
}
/**
* 根据id修改索引内容
* @param indexName
* @return
*/
public <T extends ESBaseVo> String updateIndexDataById(String indexName,String typeName,ESBaseVo eSBaseVo) {
PublicUtil.isParam(indexName,"索引名称不能为空");
PublicUtil.isParam(eSBaseVo,"对象为空");
PublicUtil.isParam(eSBaseVo.getId(),"id不能为空");
UpdateResponse update = null;
try {
UpdateRequest qyf = new UpdateRequest();
qyf.index(indexName);
qyf.type(typeName==null?esType:typeName);
qyf.id(eSBaseVo.getId());
qyf.doc(JSON.toJSONString(eSBaseVo), XContentType.JSON);
update = restHighLevelClient.update(qyf, RequestOptions.DEFAULT);
} catch (IOException e) {
log.error("根据id修改索引内容失败: "+e.getMessage());
}
return update.toString();
}
public void deleteIndexData(String indexName,String typeName,String id){
}
/**
* 批量修改索引内容
* @param indexName
* @return
*/
public <T extends ESBaseVo> String batchUpdateIndexDataById(String indexName,List<T> list) {
PublicUtil.isParam(indexName,"索引名称不能为空");
try {
BulkRequest bulkRequest = new BulkRequest();
list.forEach(data->{
UpdateRequest qyf = new UpdateRequest();
qyf.index(indexName);
qyf.type(esType);
qyf.id(data.getId());
qyf.doc(JSON.toJSONString(data), XContentType.JSON);
bulkRequest.add(qyf);
});
BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
System.out.println(bulk.hasFailures());
} catch (IOException e) {
log.error("批量修改索引内容失败: "+e.getMessage());
}
return "";
}
/**
* 根据id删除索引数据
* @param indexName
* @param id
*/
public void deleteIndexData(String indexName,String id){
PublicUtil.isParam(indexName,"索引名称不能为空");
PublicUtil.isParam(id,"id不能为空");
try {
DeleteRequest deleteRequest = new DeleteRequest(indexName,esType,id);
restHighLevelClient.delete(deleteRequest);
} catch (IOException e) {
log.error("根据id删除索引数据失败: "+e.getMessage());
}
}
public <T extends ESBaseVo> String batchInsertData(String indexName,List<T> list) {
return batchInsertData(indexName,null,list);
}
/**
* 批量插入索引数据
* @param indexName
* @return
*/
public <T extends ESBaseVo> String batchInsertData(String indexName,String typeName,List<T> list) {
try {
BulkRequest bulkRequest = new BulkRequest();
list.forEach(data->{
IndexRequest indexRequest=new IndexRequest();
indexRequest.type(typeName==null?esType:typeName);
indexRequest.id(data.getId());
indexRequest.index(indexName);
indexRequest.source(JSON.toJSONString(data), XContentType.JSON);
bulkRequest.add(indexRequest);
});
BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
System.out.println(bulk.hasFailures());
} catch (IOException e) {
e.printStackTrace();
}
return "success";
}
/**
*通过分页查询数据
* @param indexName
* @param pageSize
* @param currentPage
* @return
*/
public <T> PageInfo<T> getPageInfoData(String indexName, int pageSize, int currentPage, Class<T> clazz, Object queryObj){
if(pageSize>10000){
return new PageInfo<>();
}
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
ESRestSearchUtils.dealBoolBuilder(searchSourceBuilder,boolQueryBuilder,queryObj);
searchSourceBuilder.query(boolQueryBuilder).size(pageSize);
searchSourceBuilder.fetchSource("*","");
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
int toatal = this.scrollSearchAllCount(searchRequest);
if(toatal > 0){
//游标分页查询
SearchResponse searchResponse= scrollSearchAllByPage(currentPage, pageSize, searchRequest);
SearchHits hits = searchResponse.getHits();
List<T> jsonStringList=new ArrayList<>();
if (searchResponse != null && hits.getHits().length > 0) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
T order = JSON.parseObject(hit.getSourceAsString(), clazz);
jsonStringList.add(order);
}
}
PageInfo pageInfo = new PageInfo();
pageInfo.setList(jsonStringList);
pageInfo.setPageNum(currentPage);
pageInfo.setPageSize(pageSize);
pageInfo.setTotal(toatal);
return pageInfo;
}else{
return new PageInfo<>();
}
}
/**
* @Description 查询esList
* @Author
* @Date 2020/12/18 13:44
* @Param
* @return
*/
public <T> List<T> getEsListData(String indexName, int pageSize, int currentPage, Class<T> clazz, Object queryObj) {
List<T> jsonStringList=new ArrayList<>();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
ESRestSearchUtils.dealBoolBuilder(searchSourceBuilder,boolQueryBuilder,queryObj);
searchSourceBuilder.query(boolQueryBuilder).size(pageSize);
searchSourceBuilder.fetchSource("*","");
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
//游标分页查询
SearchResponse searchResponse = scrollSearchAllByPage(currentPage, pageSize, searchRequest);
SearchHits hits = searchResponse.getHits();
if (searchResponse != null && hits.getHits().length > 0) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
T order = JSON.parseObject(hit.getSourceAsString(), clazz);
jsonStringList.add(order);
}
}
return jsonStringList;
}
/**
* 根据条件查询所有数据（不分页）
* @param indexName
* @param clazz
* @param queryObj
* @param <T>
* @return
*/
public <T> List<T> getEsInfoData(String indexName, Class<T> clazz, Object queryObj){
List<T> jsonStringList=new ArrayList<>();
try {
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(3L));
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
ESRestSearchUtils.dealBoolBuilder(searchSourceBuilder,boolQueryBuilder,queryObj);
searchSourceBuilder.query(boolQueryBuilder).size(defaultPageSize);
searchSourceBuilder.fetchSource("*","");
searchSourceBuilder.trackTotalHits(true);
// 超时时间120s
searchSourceBuilder.timeout(new TimeValue(120, TimeUnit.SECONDS));
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
// 开启scroll查询,设置scroll过期时间为1min
searchRequest.scroll(scroll);
// 发起请求并接收响应
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 获取第一页的查询结果
SearchHit[] searchHits=searchResponse.getHits().getHits();
for (SearchHit hit : searchHits) {
T order = JSON.parseObject(hit.getSourceAsString(), clazz);
jsonStringList.add(order);
}
// 获取ScrollId
String scrollId = searchResponse.getScrollId();
// 返回结果不为空则滚动查询
while (searchHits != null && searchHits.length > 0){
// 初始化scroll查询
SearchScrollRequest searchScrollRequest=new SearchScrollRequest(scrollId);
searchScrollRequest.scroll(scroll);
// 发起请求并接收响应
searchResponse=this.restHighLevelClient.scroll(searchScrollRequest,RequestOptions.DEFAULT);
// 更新ScrollId
scrollId=searchResponse.getScrollId();
// 更新查询结果
searchHits=searchResponse.getHits().getHits();
// 放入List
for (SearchHit hit : searchHits) {
T order = JSON.parseObject(hit.getSourceAsString(), clazz);
jsonStringList.add(order);
}
}
//释放游标
if (scrollId != null) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
}
} catch (IOException e) {
log.error("getEsInfoData fail: "+e.getMessage());
}
return jsonStringList;
}
/**
* 单个添加索引内容（根据id）
* @param indexName
* @return
*/
public <T> Wrapper insertData(String indexName,String indexType,T t) {
try {
Class<?> cl = t.getClass();
Field[] fs = cl.getDeclaredFields();
String id = null;
for(Field f : fs){
try {
Class c = f.getType();
String filedName = f.getName().substring(0,1).toUpperCase()+f.getName().substring(1);
if("SerialVersionUID".equalsIgnoreCase(filedName)) {
continue;
}
Method me = cl.getDeclaredMethod("get"+filedName);
Object o = me.invoke(t);
if(filedName.equals("Id")){
id = o.toString();
break;
}
} catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
log.error("insertDatas error{}",e);
}
}
IndexRequest request = new IndexRequest(indexName);
request.source(JSON.toJSONString(t), XContentType.JSON);
request.id(id==null?SnowFlake.nextStrId():id);
request.type(PublicUtil.isEmpty(indexType)?esType:indexType);
log.info("indexRequest type:{}, id:{} ,source: {}",request.type(),request.id(),request.source());
restHighLevelClient.index(request, RequestOptions.DEFAULT);
return (Wrapper) WrapMapper.ok();
} catch (IOException e) {
log.error("添加索引内容失败: "+e.getMessage());
}
return (Wrapper) WrapMapper.ok();
}
public <T>Wrapper updateEsData(String indexName,String indexType, T t,String id) throws Exception {
UpdateResponse update = null;
try {
UpdateRequest qyf = new UpdateRequest();
qyf.index(indexName);
qyf.type(PublicUtil.isEmpty(indexType)?esType:indexType);
qyf.id(id);
qyf.doc(JSON.toJSONString(t), XContentType.JSON);
restHighLevelClient.update(qyf, RequestOptions.DEFAULT);
return WrapMapper.ok();
} catch (IOException e) {
log.error("根据id修改索引内容失败: "+e.getMessage());
}
return WrapMapper.ok();
}
/**
* @Description 查询es总数
* @Author
* @Date 2020/12/16 18:26
* @Param
* @return
*/
public int searchEsCount(String indexName, Object queryObj) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
ESRestSearchUtils.dealBoolBuilder(searchSourceBuilder,boolQueryBuilder,queryObj);
searchSourceBuilder.query(boolQueryBuilder);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
return this.scrollSearchAllCount(searchRequest);
}
/**
* 执行搜索（带分组）
* @param indexName 索引名称
* @param queryObj 复合过滤
* @param groupParam 要分组的参数
*
*/
public Map<String, String> getGroupCount(String indexName,Object queryObj,String groupParam) {
Map<String, String> dataMap = new HashMap<>();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 复合过滤器
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
ESRestSearchUtils.dealBoolBuilder(searchSourceBuilder,boolQueryBuilder,queryObj);
searchSourceBuilder.query(boolQueryBuilder).size(defaultPageSize);
searchSourceBuilder.fetchSource("*","");
searchSourceBuilder.trackTotalHits(true);
// 根据institutionId分组统计总数和实际得分
AggregationBuilder termsBuilder=AggregationBuilders.terms("group_by").field(groupParam).size(20000);
//termsBuilder.subAggregation(AggregationBuilders.terms("group_by_orderType").field("orderType"));
searchSourceBuilder.aggregation(termsBuilder);
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse=restHighLevelClient.search(searchRequest,RequestOptions.DEFAULT);
Aggregations aggregations=searchResponse.getAggregations();
Terms terms=aggregations.get("group_by");
terms.getBuckets().forEach( te ->{
dataMap.put(te.getKey().toString(),te.getDocCount()+"");
});
} catch (IOException e) {
e.printStackTrace();
}
//解析返回数据，获取分组名称为aggs-class的数据
return dataMap;
}
/**
* 根据index删除
* @param index
*/
public void deleteIndex(String index){
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index);
try {
AcknowledgedResponse deleteResponse = restHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
boolean acknowledged = deleteResponse.isAcknowledged();
if (acknowledged) {
return ;
}
} catch (IOException e) {
log.error("es delete index failed{}", e.getCause());
throw new BusinessException(e.getCause());
}
}
}
/**
* 查询封装类
*/
public class ESRestSearchUtils {
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
/**
* 查询
* @param sourceBuilder
* @param boolBuilder
* @param queryObj
*/
public static void dealBoolBuilder(SearchSourceBuilder sourceBuilder, BoolQueryBuilder boolBuilder,Object queryObj) {
buildBoolQuery(sourceBuilder,boolBuilder,queryObj );
}
private static BoolQueryBuilder buildBoolQuery(SearchSourceBuilder sourceBuilder,BoolQueryBuilder boolQueryBuilder,Object queryObj) {
/**
* 匹配查询
*/
buildMatchQuery(boolQueryBuilder, queryObj);
/**
* 范围查询
*/
buildRangeQuery(boolQueryBuilder, queryObj);
/**
* 排序
*/
buildSortQuery(sourceBuilder,queryObj);
BoolQueryBuilder queryBuilder = boolQuery().must(boolQueryBuilder);
return queryBuilder;
}
/**
* 范围查询构建
*
* @param boolQueryBuilder
* @param query
*/
private static void buildRangeQuery(BoolQueryBuilder boolQueryBuilder, Object query) {
Class clazz = query.getClass();
for (Field field : clazz.getDeclaredFields()) {
IndexSearchType annotation = field.getAnnotation(IndexSearchType.class);
Object value = getFieldValue(field, query);
if (annotation == null || PublicUtil.isEmpty(value)) {
continue;
}
if (IndexSearchTypeConst.gt.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).gt(formatValue(value)));
} else if (IndexSearchTypeConst.gte.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).gte(formatValue(value)));
} else if (IndexSearchTypeConst.lt.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).lt(formatValue(value)));
} else if (IndexSearchTypeConst.lte.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).lte(formatValue(value)));
} else if (IndexSearchTypeConst.date_from.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).from(formatValue(value)));
} else if (IndexSearchTypeConst.date_to.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).to(formatValue(value)));
}
}
}
private static void buildSortQuery(SearchSourceBuilder searchSourceBuilder, Object query) {
Class clazz = query.getClass();
for (Field field : clazz.getDeclaredFields()) {
IndexSearchType annotation = field.getAnnotation(IndexSearchType.class);
if (annotation == null || PublicUtil.isEmpty(annotation.sortType())) {
continue;
}
String fieldName= getFieldName(field, annotation.column());
if(annotation.sortType().equals(SortOrder.DESC.toString())){
searchSourceBuilder.sort(fieldName,SortOrder.DESC);
}else{
searchSourceBuilder.sort(fieldName,SortOrder.ASC);
}
}
}
/**
* and or 查询构建
*
* @param boolQueryBuilder
* @param query
*/
private static void buildMatchQuery(BoolQueryBuilder boolQueryBuilder, Object query) {
Class clazz = query.getClass();
for (Field field : clazz.getDeclaredFields()) {
IndexSearchType annotation = field.getAnnotation(IndexSearchType.class);
Object value = getFieldValue(field, query);
if (annotation == null || PublicUtil.isEmpty(value)) {
continue;
}
if (IndexSearchTypeConst.gt.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).gt(formatValue(value)));
} else if (IndexSearchTypeConst.gte.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).gte(formatValue(value)));
} else if (IndexSearchTypeConst.lt.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).lt(formatValue(value)));
} else if (IndexSearchTypeConst.lte.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).lte(formatValue(value)));
} else if (IndexSearchTypeConst.date_from.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).from(formatValue(value)));
} else if (IndexSearchTypeConst.date_to.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.rangeQuery(getFieldName(field, annotation.column())).to(formatValue(value)));
}
}
}
private static void buildSortQuery(SearchSourceBuilder searchSourceBuilder, Object query) {
Class clazz = query.getClass();
for (Field field : clazz.getDeclaredFields()) {
IndexSearchType annotation = field.getAnnotation(IndexSearchType.class);
if (annotation == null || PublicUtil.isEmpty(annotation.sortType())) {
continue;
}
String fieldName= getFieldName(field, annotation.column());
if(annotation.sortType().equals(SortOrder.DESC.toString())){
searchSourceBuilder.sort(fieldName,SortOrder.DESC);
}else{
searchSourceBuilder.sort(fieldName,SortOrder.ASC);
}
}
}
/**
* and or 查询构建
*
* @param boolQueryBuilder
* @param query
*/
private static void buildMatchQuery(BoolQueryBuilder boolQueryBuilder, Object query) {
Class clazz = query.getClass();
for (Field field : clazz.getDeclaredFields()) {
IndexSearchType annotation = field.getAnnotation(IndexSearchType.class);
Object value = getFieldValue(field, query);
if (annotation == null || PublicUtil.isEmpty(value)) {
continue;
}
String fieldName=getFieldName(field, annotation.column());
if (IndexSearchTypeConst.match.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery(fieldName, "*" + formatValue(value).toString() + "*"));
} else if (IndexSearchTypeConst.term.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.termQuery(fieldName, formatValue(value).toString()));
} else if (IndexSearchTypeConst.multiple.equals(annotation.type())) {
if (value instanceof Collection) {//集合list
Collection tmp = (Collection) value;
boolQueryBuilder.must(QueryBuilders.termsQuery(fieldName, tmp));
} else if (value.getClass().isArray()) {//数组[]
Object[] objArr = (Object[]) value;
if (objArr.length > 0) {
List<Object> tmp = Arrays.stream(objArr).collect(Collectors.toList());
boolQueryBuilder.must(QueryBuilders.termsQuery(fieldName, tmp));
}
} else {
boolQueryBuilder.must(QueryBuilders.termQuery(fieldName, formatValue(value)));
}
} else if (IndexSearchTypeConst.id.equals(annotation.type())) {
IdsQueryBuilder queryBuilder = QueryBuilders.idsQuery();
if (value instanceof Collection) {
Collection tmp = (Collection) value;
for (Object obj : tmp) {
queryBuilder.addIds(obj.toString());
}
boolQueryBuilder.must(queryBuilder);
} else {
queryBuilder.addIds(value.toString());
boolQueryBuilder.must(queryBuilder);
}
}else if(IndexSearchTypeConst.likeOr.equals(annotation.type())) {
BoolQueryBuilder rueryBuilder = QueryBuilders.boolQuery();
Class childObject= (Class) value.getClass();
Object object2=formatValue(value);
for (Field childField : childObject.getDeclaredFields()) {
IndexSearchType childNotation = childField.getAnnotation(IndexSearchType.class);
Object childValue = getFieldValue(childField,object2 );
if (childNotation == null || childValue == null) {
continue;
}
Collection tmp = (Collection) childValue;
for (Object obj : tmp) {
String childFieldName=getFieldName(childField, childNotation.column());
rueryBuilder.should(QueryBuilders.wildcardQuery(childFieldName, "*"+formatValue(obj).toString()+"*"));
}
}
boolQueryBuilder.filter(rueryBuilder);
}
else if(IndexSearchTypeConst.orIsNull.equals(annotation.type())) {
BoolQueryBuilder shouldQueryBuilder = QueryBuilders.boolQuery();
doSetObjectQuery(value, fieldName, shouldQueryBuilder);
shouldQueryBuilder.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(fieldName))); // is null
shouldQueryBuilder.should(QueryBuilders.termQuery(fieldName,"")); // = '' 空字符串
boolQueryBuilder.filter(shouldQueryBuilder);
}
else if(IndexSearchTypeConst.orNotNull.equals(annotation.type())) {
BoolQueryBuilder shouldQueryBuilder = QueryBuilders.boolQuery();
doSetObjectQuery(value, fieldName, shouldQueryBuilder);
boolQueryBuilder.filter(shouldQueryBuilder);
}
else if(IndexSearchTypeConst.exist.equals(annotation.type())) {
boolQueryBuilder.must(QueryBuilders.existsQuery(getFieldName(field, annotation.column())));
}
else if(IndexSearchTypeConst.notExist.equals(annotation.type())) {
boolQueryBuilder.mustNot(QueryBuilders.existsQuery(getFieldName(field, annotation.column())));
}
else if(IndexSearchTypeConst.gtInWwoWords.equals(annotation.type())) {
String fieldValue= formatValue(value).toString();
Script script = new Script(ScriptType.INLINE,"painless","Integer.parseInt(String.valueOf(null == doc['"+fieldName+"'].value ? 0 : doc['"+fieldName+"'].value)) > Integer.parseInt(String.valueOf(null == doc['"+fieldValue+"'].value ? 0 : doc['"+fieldValue+"'].value))", new HashMap<>());
ScriptQueryBuilder scriptQueryBuilder = new ScriptQueryBuilder(script);
boolQueryBuilder.must(scriptQueryBuilder);
}
else if(IndexSearchTypeConst.lteInTwoWords.equals(annotation.type())) {
String fieldValue= formatValue(value).toString();
Script script = new Script(ScriptType.INLINE,"painless","Integer.parseInt(String.valueOf(null == doc['"+fieldName+"'].value ? 0 : doc['"+fieldName+"'].value)) <= Integer.parseInt(String.valueOf(null == doc['"+fieldValue+"'].value ? 0 : doc['"+fieldValue+"'].value))", new HashMap<>());
ScriptQueryBuilder scriptQueryBuilder = new ScriptQueryBuilder(script);
boolQueryBuilder.must(scriptQueryBuilder);
}
else if(IndexSearchTypeConst.commaStr.equals(annotation.type())) {
//匹配逗号隔开的字符串,例如：java参数 x;ES字段值x,y,z 传x或者y或者z进来则可以查出结果
String fieldValue= formatValue(value).toString();
BoolQueryBuilder boolQueryBuilderCommaStr = QueryBuilders.boolQuery();
boolQueryBuilderCommaStr.should(QueryBuilders.wildcardQuery(fieldName,fieldValue));
boolQueryBuilderCommaStr.should(QueryBuilders.wildcardQuery(fieldName,"*,"+fieldValue));
boolQueryBuilderCommaStr.should(QueryBuilders.wildcardQuery(fieldName,fieldValue+",*"));
boolQueryBuilderCommaStr.should(QueryBuilders.wildcardQuery(fieldName,"*,"+fieldValue+",*"));
boolQueryBuilder.filter(boolQueryBuilderCommaStr);
}
else if(IndexSearchTypeConst.script.equals(annotation.type())) {
String fieldValue= formatValue(value).toString();
Script script = new Script(ScriptType.INLINE,"painless",fieldValue, new HashMap<>());
ScriptQueryBuilder scriptQueryBuilder = new ScriptQueryBuilder(script);
boolQueryBuilder.must(scriptQueryBuilder);
}
else if (IndexSearchTypeConst.should.equals(annotation.type())) {
if (value instanceof Collection) {
BoolQueryBuilder shouldQueryBuilder = boolQuery();
Collection tmp = (Collection) value;
for (Object obj : tmp) {
shouldQueryBuilder.should(matchQuery(getFieldName(field, annotation.column()), formatValue(obj)));
}
boolQueryBuilder.should(shouldQueryBuilder);
} else {
boolQueryBuilder.should(boolQuery().should(matchQuery(getFieldName(field, annotation.column()), formatValue(value))));
}
}
}
}
/**
* 设置集合信息
* @param value
* @param fieldName
* @param shouldQueryBuilder
*/
private static void doSetObjectQuery(Object value, String fieldName, BoolQueryBuilder shouldQueryBuilder) {
if (value instanceof Collection) {
Collection tmp = (Collection) value;
for (Object obj : tmp) {
shouldQueryBuilder.should(QueryBuilders.termQuery(fieldName, formatValue(obj)));
}
} else {
shouldQueryBuilder.must(QueryBuilders.termQuery(fieldName, formatValue(value)));
}
}
/**
* 获取字段值
*
* @param field
* @param obj
* @return
*/
protected static Object getFieldValue(Field field, Object obj) {
boolean isAccessible = field.isAccessible();
field.setAccessible(true);
try {
return field.get(obj);
} catch (IllegalAccessException e) {
throw new BusinessException(e);
} finally {
field.setAccessible(isAccessible);
}
}
/**
* 获取字段名，若设置column则返回该值
*
* @param field
* @param column
* @return
*/
protected static String getFieldName(Field field, String column) {
return StringUtils.isNotBlank(column) ? column : field.getName();
}
/**
* 转换为es识别的value值
*
* @param value
* @return
*/
protected static Object formatValue(Object value) {
if (value instanceof Date) {
return sdf.format(value);
} else {
return value;
}
}
}
使用
<dependency>
<groupId>com.*.paascloud.common</groupId>
<artifactId>paascloud-common-es</artifactId>
<version>1.0</version>
</dependency>vo对象
@Data
public class EsUserCommentVo extends ESBaseVo {
@IndexType(type = IndexTypeConst.TEXT)
private String companyId;
@IndexType(type = IndexTypeConst.TEXT)
private String orgId;
@IndexType(type = IndexTypeConst.TEXT)
private String xxxx;
}调用工具类
@Autowired
private ESRestClientUtils esRestClientUtils;
@Slf4j
@Service
public class aaa{
@Override
public String createIndex(String companyId) {
Calendar cal=Calendar.getInstance();
int year=cal.get(Calendar.YEAR);
String index = EsIndexConstant.***_USER_COMMENT + companyId+"_"+year;
log.info("index is---------------" + index);
if (!esRestClientUtils.existIndex(index)) {
esRestClientUtils.saveIndex(index, **.class);
}
return index;
}
@Override
public void insertDateToEs(String startTime, String endTime) {
log.info("*******-------------------");
List<String> ids = ***CommentMapper.selectCompanyListIds(startTime, endTime);
if (PublicUtil.isEmpty(ids)) {
return;
}
log.info("同步评论记录"+ids.size());
List<String> listIds=new ArrayList<>();
for (String complanyId : ids) {
String index = createIndex(complanyId);
List<dsa> list = ***CommentMapper.selectDataList(complanyId,startTime, endTime);
List<dsa> ***l = new ArrayList<>();
list.stream().forEach(o -> {
dsa *** = new dsa();
esUserSituationVos.add(***);
listIds.add(o.getId());
});
esRestClientUtils.batchInsertData(index,***l);
}
log.info("删除同步数据记录"+delelteCount);
log.info("******同步结束-------------------");
}
}
3684
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
