项目中用到了es,通过ELK存储日志,然后通过系统界面查询,对此es常用操作做了封装,不多说,直接上代码。
环境:jdk(1.8)+es(7.15.2)+springboot(2.3.5.RELEASE)
1、pom.xml
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.15.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.6.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.6.2</version>
</dependency>
2、EsRestClientConfig.java
package net.wfl.framework.boot.tools.es;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
/**
* ES配置类
*
* @author wangfenglei
*/
@ConditionalOnProperty(value = "wfl.base.tools.es.switch", havingValue = "true", matchIfMissing = false)
@Component
public class EsRestClientConfig {
@Value("${wfl.base.tools.es.ip}")
private String hostname;
@Value("${wfl.base.tools.es.port}")
private int port;
@Value("${wfl.base.tools.es.username}")
private String username;
@Value("${wfl.base.tools.es.password}")
private String password;
/**
* 创建es restHighLevelClient
*
* @return 客户端
*/
@ConditionalOnProperty(value = "wfl.base.tools.es.switch", havingValue = "true", matchIfMissing = false)
@Bean
public RestHighLevelClient restHighLevelClient() {
final CredentialsProvider credentialsProvider =
new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(username, password));
RestClientBuilder builder = RestClient.builder(
new HttpHost(hostname, port))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(
HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder
.setDefaultCredentialsProvider(credentialsProvider);
}
});
return new RestHighLevelClient(builder);
}
}
3、HighLevelRestUtil.java
package net.wfl.framework.boot.tools.es;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.ParameterizedType;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* ES工具类
*
* @author wangfenglei
*/
@Slf4j
@Component
@ConditionalOnProperty(value = "wfl.base.tools.es.switch", havingValue = "true", matchIfMissing = false)
public class HighLevelRestUtil {
@Autowired
private RestHighLevelClient restHighLevelClient;
/**
* 判断索引是否存在
*
* @param index 索引
* @return 成功标识
*/
public boolean checkIndex(String index) {
try {
return restHighLevelClient.indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT);
} catch (Exception e) {
log.error(e.toString(), e);
return Boolean.FALSE;
}
}
/**
* 创建索引
*
* @param indexName 索引名称
* @param columnMap 列
* @return 创建索引成功标识
*/
public boolean createIndex(String indexName, Map<String, Object> columnMap) {
try {
if (!checkIndex(indexName)) {
CreateIndexRequest request = new CreateIndexRequest(indexName);
if (columnMap != null && columnMap.size() > 0) {
Map<String, Object> source = new HashMap<>();
source.put("properties", columnMap);
request.mapping(source);
}
this.restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
}
return Boolean.TRUE;
} catch (Exception e) {
log.error(e.toString(), e);
return Boolean.FALSE;
}
}
/**
* 删除索引
*
* @param indexName 索引名称
* @return 删除成功标识
*/
public boolean deleteIndex(String indexName) {
try {
if (checkIndex(indexName)) {
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
AcknowledgedResponse response = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
return response.isAcknowledged();
}
} catch (Exception e) {
log.error(e.toString(), e);
}
return Boolean.FALSE;
}
/**
* 获取ES信息
*
* @return Es信息
*/
public SearchResponse getEsInfo() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// SearchRequest
SearchRequest searchRequest = new SearchRequest();
searchRequest.source(searchSourceBuilder);
// 查询ES
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
return searchResponse;
}
/**
* 获取总数
*
* @param indexList 索引
* @param queryBuilder 查询条件
* @return 总数
* @throws Exception 异常
*/
public long getCount(List<String> indexList, QueryBuilder queryBuilder) throws Exception {
// 通过CountRequest查询获得count
CountRequest countRequest = new CountRequest();
if (null != indexList) {
// 绑定索引名
countRequest.indices(indexList.toArray(new String[indexList.size()]));
}
countRequest.query(queryBuilder);
CountResponse countResponse = restHighLevelClient.count(countRequest, RequestOptions.DEFAULT);
// 获取总数
long total = countResponse.getCount();
return total;
}
/**
* 从ES分页查询数据列表
*
* @param indexList 索引列表
* @param page 分页信息
* @param keyword 关键字
* @param clazz 类
* @param orderField 排序字段
* @param fieldNames 查询字段
* @param <T> 泛型
* @return 数据列表
* @throws Exception 异常
*/
public <T> Page<T> pageList(List<String> indexList, Page<T> page, String keyword, Class<T> clazz, String orderField, String... fieldNames) throws Exception {
if (StringUtils.isEmpty(keyword)) {
throw new Exception("The keywork is blank!");
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 分页采用简单的from + size分页,适用数据量小的,了解更多分页方式可自行查阅资料
searchSourceBuilder.from((int) ((page.getCurrent() - 1) * page.getSize())).size((int) page.getSize());
QueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword, fieldNames);
searchSourceBuilder.query(queryBuilder);
long total = getCount(indexList, queryBuilder);
if (StringUtils.isNotEmpty(orderField)) {
// 排序,根据ID倒叙
searchSourceBuilder.sort(orderField, SortOrder.DESC);
}
// SearchRequest
SearchRequest searchRequest = null == indexList ? new SearchRequest() : new SearchRequest(indexList.toArray(new String[indexList.size()]));
searchRequest.source(searchSourceBuilder);
// 查询ES
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
// 遍历封装列表对象
List<T> resultList = new ArrayList<>();
SearchHit[] searchHits = hits.getHits();
for (SearchHit searchHit : searchHits) {
resultList.add(JSON.parseObject(searchHit.getSourceAsString(), clazz));
}
Page<T> resultPage = new Page<>(page.getCurrent(), page.getSize());
resultPage.setTotal(total);
resultPage.setRecords(resultList);
return resultPage;
}
/**
* 从ES分页查询数据列表
*
* @param indexList 索引列表
* @param page 分页信息
* @param queryBuilder 查询条件
* @param orderField 排序字段
* @param <T> 泛型
* @return 分页信息
* @throws Exception 异常
*/
public <T> Page<T> pageList(List<String> indexList, Page<T> page, QueryBuilder queryBuilder, String orderField, Class<T> clazz) throws Exception {
// 获取总数
long total = getCount(indexList, queryBuilder);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 分页采用简单的from + size分页,适用数据量小的,了解更多分页方式可自行查阅资料
searchSourceBuilder.from((int) ((page.getCurrent() - 1) * page.getSize())).size((int) page.getSize());
//设置查询条件
searchSourceBuilder.query(queryBuilder);
if (StringUtils.isNotEmpty(orderField)) {
// 排序,倒叙
searchSourceBuilder.sort(orderField, SortOrder.DESC);
}
// SearchRequest
SearchRequest searchRequest = null == indexList ? new SearchRequest() : new SearchRequest(indexList.toArray(new String[indexList.size()]));
searchRequest.source(searchSourceBuilder);
// 查询ES
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
// 遍历封装列表对象
List<T> resultList = new ArrayList<>();
SearchHit[] searchHits = hits.getHits();
for (SearchHit searchHit : searchHits) {
resultList.add(JSON.parseObject(searchHit.getSourceAsString(), clazz));
}
Page<T> resultPage = new Page<>(page.getCurrent(), page.getSize());
resultPage.setTotal(total);
resultPage.setRecords(resultList);
return resultPage;
}
/**
* 查询数据列表,不分页,慎用
*
* @param indexList 索引列表
* @param queryBuilder 查询条件
* @param <T> 泛型
* @return 数据列表
* @throws Exception 异常
*/
public <T> List<T> getList(List<String> indexList, QueryBuilder queryBuilder) throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(queryBuilder);
// SearchRequest
SearchRequest searchRequest = null == indexList ? new SearchRequest() : new SearchRequest(indexList.toArray(new String[indexList.size()]));
searchRequest.source(searchSourceBuilder);
// 查询ES
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
// 遍历封装列表对象
List<T> resultList = new ArrayList<>();
SearchHit[] searchHits = hits.getHits();
for (SearchHit searchHit : searchHits) {
resultList.add(JSON.parseObject(searchHit.getSourceAsString(), (Class<T>) ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[0]));
}
return resultList;
}
/**
* 根据ID查询
*
* @param index 索引
* @param id id
* @return 返回数据
*/
public <T> T getById(String index, String id) throws Exception {
// GetRequest
GetRequest getRequest = null == index ? new GetRequest() : new GetRequest(index, id);
// 查询ES
GetResponse getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
T result = JSON.parseObject(getResponse.getSourceAsString(), (Class<T>) ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[0]);
return result;
}
/**
* 保存文档
*
* @param index 索引
* @param entry 数据类
* @param <T> 泛型
* @return 是否成功
* @throws Exception 异常
*/
public <T> int save(String index, T entry) throws Exception {
if (null == index) {
throw new Exception("The index is null!");
}
// IndexRequest
IndexRequest indexRequest = new IndexRequest(index);
String id = getId(entry);
String source = JSON.toJSONString(entry);
indexRequest.id(id).source(source, XContentType.JSON);
// 操作ES
IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
return indexResponse.status().getStatus();
}
/**
* 保存文档
*
* @param index 索引
* @param entryList 数据列表
* @param <T> 泛型
* @return 是否成功
* @throws Exception 异常
*/
public <T> int saveBatch(String index, List<T> entryList) throws Exception {
if (null == index) {
throw new Exception("The index is null!");
}
BulkRequest request = new BulkRequest();
String id;
for (T entry : entryList) {
id = getId(entry);
request.add(new IndexRequest(index).id(id).source(JSON.toJSONString(entry), XContentType.JSON));
}
//设置刷新策略
request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
// 操作ES批量插入,此为同步写入,如果存在性能问题,可考虑自行封装异步批量插入
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
return bulkResponse.status().getStatus();
}
/**
* 修改数据
*
* @param index 索引
* @param entry 数据类型
* @param <T> 数据类型
* @return 修改成功标识
* @throws Exception 异常
*/
public <T> int update(String index, T entry) throws Exception {
if (null == index) {
throw new Exception("The index is null!");
}
Field field = entry.getClass().getDeclaredField("id");
if (null == field) {
throw new Exception(" The class is not exist id!");
}
field.setAccessible(true);
String id = field.get(entry).toString();
UpdateRequest updateRequest = new UpdateRequest(index, id);
updateRequest.doc(JSON.toJSONString(entry), XContentType.JSON);
// 操作ES
UpdateResponse updateResponse = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
return updateResponse.status().getStatus();
}
/**
* 删除文档
*
* @param index 索引
* @param id id
* @return 删除成功标识
* @throws Exception 异常
*/
public int deleteById(String index, String id) throws Exception {
if (null == index) {
throw new Exception("The index is null!");
}
// DeleteRequest
DeleteRequest deleteRequest = new DeleteRequest(index, id);
// 操作ES
DeleteResponse deleteResponse = restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
return deleteResponse.status().getStatus();
}
/**
* 根据添加批量删除
*
* @param indexList 索引列表
* @param queryBuilder 查询条件
* @return 删除条数
* @throws Exception 异常
*/
public long deleteByQuery(List<String> indexList, QueryBuilder queryBuilder) throws Exception {
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest();
if (CollectionUtils.isNotEmpty(indexList)) {
deleteRequest.indices(indexList.toArray(new String[indexList.size()]));
}
deleteRequest.setQuery(queryBuilder);
// 操作ES删除
BulkByScrollResponse response = restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT);
return response.getStatus().getDeleted();
}
/**
* 获取ID
*
* @param entry 数据类
* @param <T> 泛型
* @return ID
* @throws Exception 异常
*/
private <T> String getId(T entry) {
try {
Field field = entry.getClass().getDeclaredField("id");
if (null == field || null == field.get(entry)) {
return IdWorker.getIdStr();
} else {
field.setAccessible(true);
return field.get(entry).toString();
}
} catch (Exception e) {
return IdWorker.getIdStr();
}
}
}
说明:实现的分页查询使用的是from+size查询,对于深度查询和获取大量结果查询会有比较严重的问题。深度查询请使用search after查询,大量结果获取请使用scroll查询,用到可以自行封装。