ES依赖pom配置
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.10.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.10.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.10.1</version>
</dependency>
ElasticSearchConfig 文件配置
package com.xx.xx.webservices.config;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.ArrayList;
import java.util.List;
/**
* @Author : 骑士李四
* @Date : Created in 10:29 2021/11/10
* @Description :
* @since 3.0
*/
@Configuration
public class ElasticSearchConfig {
/** 协议 */
@Value("${elasticsearch.schema:http2}")
private String schema;
/** 集群地址,如果有多个用“,”隔开 */
@Value("${elasticsearch.address}")
private String address;
/** 连接超时时间 */
@Value("${elasticsearch.connectTimeout}")
private int connectTimeout;
/** Socket 连接超时时间 */
@Value("${elasticsearch.socketTimeout}")
private int socketTimeout;
/** 获取连接的超时时间 */
@Value("${elasticsearch.connectionRequestTimeout}")
private int connectionRequestTimeout;
/** 最大连接数 */
@Value("${elasticsearch.maxConnectNum}")
private int maxConnectNum;
/** 最大路由连接数 */
@Value("${elasticsearch.maxConnectPerRoute}")
private int maxConnectPerRoute;
@Value("${elasticsearch.userName}")
private String userName;
@Value("${elasticsearch.password}")
private String password;
@Bean
public RestHighLevelClient restHighLevelClient() {
// 拆分地址
List<HttpHost> hostLists = new ArrayList<>();
String[] hostList = address.split(",");
for (String addr : hostList) {
String host = addr.split(":")[0];
String port = addr.split(":")[1];
hostLists.add(new HttpHost(host, Integer.parseInt(port), schema));
}
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
/*设置账号密码*/
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName,password));
// 转换成 HttpHost 数组
HttpHost[] httpHost = hostLists.toArray(new HttpHost[]{});
// 构建连接对象
RestClientBuilder builder = RestClient.builder(httpHost);
// 异步连接延时配置
builder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(connectTimeout);
requestConfigBuilder.setSocketTimeout(socketTimeout);
requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeout);
return requestConfigBuilder;
});
// 异步连接数配置
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnTotal(maxConnectNum);
httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpClientBuilder;
});
return new RestHighLevelClient(builder);
}
}
propertity文件配置
elasticsearch.schema=http
elasticsearch.address=xxxxx:9200
elasticsearch.connectTimeout=20000
elasticsearch.socketTimeout=20000
elasticsearch.connectionRequestTimeout=20000
elasticsearch.maxConnectNum=100
elasticsearch.maxConnectPerRoute=100
elasticsearch.userName=xxx
elasticsearch.password=xxx
在这里插入代码片
ESUtils
package com.hundsun.lightdata.webservices.utils;
import com.alibaba.fastjson.JSON;
import com.hundsun.jrescloud.common.exception.BaseBizException;
import com.hundsun.lightdata.common.entity.common.EsUdataDoc;
import com.hundsun.lightdata.common.enumpackage.ResponseEnum;
import com.lightdata.provide.repsonse.QuantizationFactorToEsResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @Author : 骑士李四
* @Date : Created in 15:53 2021/11/10
* @Description : es工具类
*/
@Slf4j
@Component
public class EsUtils {
@Autowired
private RestHighLevelClient restHighLevelClient;
private static RestHighLevelClient staticRestHighLevelClient;
@PostConstruct
public void init() {
staticRestHighLevelClient = this.restHighLevelClient;
}
/**
* 创建索引
* @author lipeng39232
* @param index
* @throws Exception
*/
public static void createIndex(String index) throws Exception {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(index)
.settings(
Settings.builder()
.put("number_of_shards", 6)
.put("number_of_replicas", 1)
.build()
);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
{
builder.startObject("properties");
{
//设置属性
builder.startObject("factorName");
{
builder.field("type", "text")
.field("analyzer", "ik_max_word")
.field("search_analyzer", "ik_smart");
}
builder.endObject();
builder.startObject("factorType");
{
builder.field("type", "text");
}
builder.endObject();
builder.startObject("factorSecondType");
{
builder.field("type", "text");
}
builder.endObject();
builder.startObject("stockCode");
{
builder.field("type", "text")
.field("analyzer", "ik_max_word")
.field("search_analyzer", "ik_smart");
}
builder.endObject();
builder.startObject("factorValue");
{
builder.field("type", "text");
}
builder.endObject();
builder.startObject("date");
{
builder.field("type", "date");
}
builder.endObject();
}
builder.endObject();
}
builder.endObject();
createIndexRequest.mapping(builder);
CreateIndexResponse response = staticRestHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
log.info(response.toString());
}
/**
* 保存文档
* <p>
* 如果文档存在,则更新文档;如果文档不存在,则保存文档,保存时会全字段更新
*
* @param esUdataDoc 文档数据
*/
public static void save(String index, EsUdataDoc esUdataDoc) throws IOException {
IndexRequest indexRequest = new IndexRequest(index);
Long esId = esUdataDoc.getDocumentId();
indexRequest.id(Long.toString(esId));
indexRequest.source(JSON.toJSONString(esUdataDoc), XContentType.JSON);
// 保存文档数据
staticRestHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
}
/**
* 更新文档 仅更新修改字段,其它字段保持原值
*
* @param index
* @param esUdataDoc
* @throws IOException
*/
public static void update(String index, EsUdataDoc esUdataDoc) throws IOException {
Long esId = esUdataDoc.getDocumentId();
UpdateRequest updateRequest = new UpdateRequest(index, Long.toString(esId));
updateRequest.doc(JSON.toJSONString(esUdataDoc), XContentType.JSON);
log.info("更新es数据,rq:{}", JSON.toJSONString(esUdataDoc));
staticRestHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
}
/**
* 批量更新 仅更新修改字段,其它字段保持原值
*
* @param index
* @param contentList
*/
public static void updateAll(String index, List<EsUdataDoc> contentList) throws IOException {
if (CollectionUtils.isEmpty(contentList)) {
return;
}
//批量请求
BulkRequest bulkRequest = new BulkRequest();
contentList.forEach(esUdataDoc -> {
Long esId =esUdataDoc.getDocumentId();
UpdateRequest updateRequest = new UpdateRequest(index, Long.toString(esId));
updateRequest.doc(JSON.toJSONString(esUdataDoc), XContentType.JSON);
bulkRequest.add(updateRequest);
});
staticRestHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
/**
* 批量保存文档
* <p>
* 如果集合中有些文档已经存在,则更新文档;不存在,则保存文档。
*
* @param index 索引
* @param contentList 文档集合
*/
public static void saveAll(String index, List<QuantizationFactorToEsResponse> contentList) throws IOException {
if (CollectionUtils.isEmpty(contentList)) {
return;
}
// 批量请求
BulkRequest bulkRequest = new BulkRequest();
contentList.forEach(esUdataDoc -> {
bulkRequest.add(new IndexRequest(index)
.id(esUdataDoc.getId())
.source(JSON.toJSONString(esUdataDoc), XContentType.JSON));
});
staticRestHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
/**
* 根据文档 ID 删除文档
*
* @param index 索引
* @param id 文档 ID
*/
public static void delete(String index, String id) throws IOException {
DeleteRequest deleteRequest = new DeleteRequest(index, id);
staticRestHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
}
/**
* description
* @author liZheng @Date 14:18 2021/7/29
* @param index
* @return void
*/
public static void deleteByIndex(String index) throws IOException{
if (StringUtils.isBlank(index)){
return;
}
DeleteRequest deleteRequest = new DeleteRequest();
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index);
//若无索引 不报错
deleteIndexRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
staticRestHighLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
}
/**
* 根据查询条件删除文档
*
* @param index 索引
* @param queryBuilder 查询条件构建器
*/
public static void deleteByQuery(String index, QueryBuilder queryBuilder) throws IOException {
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(index).setQuery(queryBuilder);
deleteRequest.setConflicts("proceed");
staticRestHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT);
}
/**
* 根据文档 ID 批量删除文档
*
* @param index 索引
* @param idList 文档 ID 集合
*/
public static void deleteAll(String index, List<String> idList) throws IOException {
if (CollectionUtils.isEmpty(idList)) {
return;
}
BulkRequest bulkRequest = new BulkRequest();
idList.forEach(id -> bulkRequest.add(new DeleteRequest(index, id)));
staticRestHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
/**
* 根据索引和文档 ID 获取数据
*
* @param index 索引
* @param id 文档 ID
* @param <T> 数据类型
* @return T 返回 T 类型的数据
*/
public static <T> T get(String index, String id, Class<T> resultType) throws IOException {
GetRequest getRequest = new GetRequest(index, id);
GetResponse response = staticRestHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
String resultAsString = response.getSourceAsString();
return JSON.parseObject(resultAsString, resultType);
}
/**
* 条件查询
*
* @param index 索引
* @param sourceBuilder 条件查询构建起
* @param <T> 数据类型
* @return T 类型的集合
*/
public static <T> List<Map<String,Object>> searchByQuery(String index, SearchSourceBuilder sourceBuilder, Class<T> resultType) throws IOException {
// 构建查询请求
SearchRequest searchRequest = new SearchRequest(index).source(sourceBuilder);
log.info("es条件查询**********"+searchRequest.source().toString());
// 获取返回值searchRequest = {SearchRequest@11775} "SearchRequest{searchType=QUERY_THEN_FETCH, indices=[kiplatform_library], indicesOptions=IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true], types=[], routing='null', preference='null', requestCache=null, scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null, allowPartialSearchResults=null, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, ccsMinimizeRoundtrips=true, source={"from":0,"size":10,"sort":[{"_score":{"order":"desc"}}],"highlight":{"fields":{"title":{"type":"unified"},"content":{}}}}}"
SearchResponse response = staticRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] hits = response.getHits().getHits();
// 创建空的查询结果集合
List<Map<String,Object>> results = new ArrayList<>(hits.length);
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSourceAsMap();
//处理高亮片段
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
for (Map.Entry<String, HighlightField> entry : highlightFields.entrySet()){
StringBuilder valueString = new StringBuilder();
Text[] fragments = entry.getValue().fragments();
for (Text text:fragments){
valueString.append(text);
}
source.put(entry.getKey(),valueString);
}
results.add(source);
}
return results;
}
public static SearchHits doSearch(String index, SearchSourceBuilder builder) {
SearchRequest searchRequest = new SearchRequest(index).source(builder);
try {
SearchResponse response = staticRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
return response.getHits();
} catch (IOException e) {
log.error(e.getMessage());
throw new BaseBizException(ResponseEnum.ERROR_ELASTICSEARCH.getErrorCode(), ResponseEnum.ERROR_ELASTICSEARCH.getErrorInfo());
}
}
/**
* 一个分组查询
*
* @param indexs 索引
* @param queryBuilder 条件查询构建起
* @param field 分组字段
* @return Map<String, Long> 返回的map
*/
public static Map<String, Long> getTermsAgg(QueryBuilder queryBuilder, String field, String... indexs) {
Map<String, Long> groupMap = new HashMap<>();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.size(0);
AggregationBuilder aggregationBuilder = AggregationBuilders.terms("agg").field(field);
searchSourceBuilder.aggregation(aggregationBuilder);
SearchRequest searchRequest = new SearchRequest(indexs);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
searchResponse = staticRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
log.error(e.getMessage());
throw new BaseBizException(ResponseEnum.ERROR_ELASTICSEARCH.getErrorCode(), ResponseEnum.ERROR_ELASTICSEARCH.getErrorInfo());
}
Terms terms = searchResponse.getAggregations().get("agg");
for (Terms.Bucket entry : terms.getBuckets()) {
groupMap.put(entry.getKey().toString(), entry.getDocCount());
}
return groupMap;
}
/***
* 判断当前索引是否存在
* @author lipeng
* @param indexName
* @return
* @throws IOException
*/
public static boolean isExistsIndex(String indexName) throws IOException {
//
// GetRequest getRequest = new GetRequest(indexName);
//
// boolean exists = staticRestHighLevelClient.exists(getRequest, RequestOptions.DEFAULT);
// return exists;
GetIndexRequest getRequest = new GetIndexRequest();
getRequest.indices(indexName);
boolean exists = staticRestHighLevelClient.indices().exists(getRequest, RequestOptions.DEFAULT);
return exists;
}
}