Elasticsearch 实战 - 第六讲:ES 项目实战(二):基本操作、批处理、高级查询
Elasticsearch 实战系列文章:
1:Elasticsearch 实战 - 第一讲:简介以及安装
2:Elasticsearch 实战 - 第二讲:kibana 安装以及ES 的概念名词
3:Elasticsearch 实战 - 第三讲:ES 基本操作、批处理
4:Elasticsearch 实战 - 第四讲:ES 高级查询
5:Elasticsearch 实战 - 第五讲:Java 集成 Spring Data Elasticsearch(一):简介及环境搭建
6:Elasticsearch 实战 - 第六讲:ES 项目实战(二):基本操作、批处理、高级查询
导读:上篇博客讲到了Java 集成 Spring Data Elasticsearch 的简介、环境搭建和domain 实体类的编写,本篇博客将接着讲解 如何用 Java 实现 es 基本操作、批处理、高级查询。(本文👆上部有完整的实例代码)。若有不懂的请结合之前的博客进行阅读。
以下为我编写的关于es 的基础通用功能代码,基本涵盖了es 的所有操作,之后的示例将以此基础进行讲解。
- IBaseService :提供公共的,基础的 ElasticSearch 功能
- BaseServiceImpl:IBaseService 实现类
- BaseProductEsTest:基础功能测试类
- ProductEsTest:各种高级查询功能测试类
1、接口类 IBaseService
package com.xinghua.elasticsearchservice.common.service;
import com.xinghua.elasticsearchservice.common.dto.SortDTO;
import com.xinghua.elasticsearchservice.common.model.EntityEsModel;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.Aggregations;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import java.util.List;
/**
* @Description 提供公共的,基础的 ElasticSearch 功能
* @Author 姚广星
* @Date 2020/2/24 16:57
**/
public interface IBaseService<T extends EntityEsModel> {
/**
* 获取ES索引名称
*
* @return
*/
String getIndexName();
/**
* 获取ES索引类型
*
* @return
*/
String getIndexType();
/**
* 返回泛型上的Class对象
*
* @return
*/
Class<T> getEntityClass();
/**
* 初始化数据
* 1: 删除原有的索引
* 2: 创建索引并且初始化映射
* 3: bulk 批量初始化数据
*
* @param entityList
* @throws Exception
*/
void init(List<T> entityList);
/**
* 创建索引和映射(若原有索引存在则删除重新创建)
*
* @return
*/
Boolean createEntityEsIndex();
/**
* 新增或修改
*
* @param entityModel
* @return
*/
String saveOrUpdate(T entityModel);
/**
* 删除entity
*
* @param id
* @return
*/
String delete(String id);
/**
* 删除索引
*
* @return
*/
Boolean deleteIndex();
/**
* 批量新增/更新
*
* @param entityModelList
*/
void batchInsertOrUpdate(List<T> entityModelList);
/**
* 分页查询
*
* @param nativeSearchQueryBuilder 查询条件
* @param pageNumber 页数
* @param pageSize 页码
* @return
*/
Page<T> searchPage(NativeSearchQueryBuilder nativeSearchQueryBuilder, int pageNumber, int pageSize);
/**
* 分页查询 按照指定字段排序,多个字段按照先后顺序排序
*
* @param sortDTOList 排序条件
* @param nativeSearchQueryBuilder 查询条件
* @param pageNumber 页数
* @param pageSize 页码
* @return
*/
Page<T> searchPageBySort(List<SortDTO> sortDTOList, NativeSearchQueryBuilder nativeSearchQueryBuilder, int pageNumber, int pageSize);
/**
* 查询操作
*
* @param nativeSearchQueryBuilder 查询条件
* @return
*/
List<T> searchList(NativeSearchQueryBuilder nativeSearchQueryBuilder);
/**
* 查询操作-按照指定字段排序,多个字段按照先后顺序排序
*
* @param sortDTOList 排序条件
* @param nativeSearchQueryBuilder 查询条件
* @return
*/
List<T> searchListBySort(List<SortDTO> sortDTOList, NativeSearchQueryBuilder nativeSearchQueryBuilder);
/**
* 获取PageRequest对象
*
* @param pageNumber 页数
* @param pageSize 页码
* @return
*/
PageRequest getPageRequest(int pageNumber, int pageSize);
/**
* 用于分组查询
*
* @param nativeSearchQueryBuilder 查询条件
* @return Aggregations 即 DSL 中的 aggs
*/
Aggregations query(NativeSearchQueryBuilder nativeSearchQueryBuilder);
}
2、实现类 IBaseService:
package com.xinghua.elasticsearchservice.common.service.impl;
import com.xinghua.elasticsearchservice.common.dto.SortDTO;
import com.xinghua.elasticsearchservice.common.model.EntityEsModel;
import com.xinghua.elasticsearchservice.common.utils.CommonException;
import com.xinghua.elasticsearchservice.common.service.IBaseService;
import com.xinghua.elasticsearchservice.utils.DataUtils;
import io.swagger.models.auth.In;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.*;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
/**
* @Description 提供公共的,基础的 ElasticSearch 功能
* @Author 姚广星
* @Date 2020/2/27 22:50
**/
@Slf4j
@Service
public class BaseServiceImpl<T extends EntityEsModel> implements IBaseService<T> {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Override
public String getIndexName() {
return elasticsearchTemplate.getPersistentEntityFor(getEntityClass()).getIndexName();
}
@Override
public String getIndexType() {
return elasticsearchTemplate.getPersistentEntityFor(getEntityClass()).getIndexType();
}
@Override
public Class<T> getEntityClass() {
Class<T> tClass = (Class<T>) ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[0];
log.info("The type of T : " + tClass);
return tClass;
}
@Override
public String delete(String id) {
return elasticsearchTemplate.delete(getEntityClass(), id);
}
@Override
public Boolean deleteIndex() {
return elasticsearchTemplate.deleteIndex(getEntityClass());
}
@Override
public void init(List<T> entityModelList) {
//判断索引是否存在
boolean indexExists = elasticsearchTemplate.indexExists(getEntityClass());
if (indexExists) {
//删除索引
elasticsearchTemplate.deleteIndex(getEntityClass());
}
//创建索引和mapper
this.createEntityEsIndex();
//bulk 批量初始化数据
this.batchInsertOrUpdate(entityModelList);
}
@Override
public Boolean createEntityEsIndex() {
boolean indexExists = elasticsearchTemplate.indexExists(getEntityClass());
if (indexExists) {
boolean deleteIndex = elasticsearchTemplate.deleteIndex(getEntityClass());
if (!deleteIndex) {
throw new CommonException(" delete entity elasticsearch index Error");
}
}
return this.createProductEsIndexAndMappers();
}
@Override
public void batchInsertOrUpdate(List<T> entityModelList) {
if (CollectionUtils.isEmpty(entityModelList)) {
throw new CommonException("entityModelList Can't be empty");
}
//判断索引是否存在 若不存在则创建索引和映射
if (!elasticsearchTemplate.indexExists(getEntityClass())) {
this.createEntityEsIndex();
}
List<IndexQuery> queries = new ArrayList<>();
for (T entityEsModel : entityModelList) {
IndexQuery indexQuery = new IndexQueryBuilder()
.withId(entityEsModel.getId())
.withObject(entityEsModel)
.build();
queries.add(indexQuery);
}
//批量插入
this.bulkInsert(queries);
}
@Override
public String saveOrUpdate(T entityModel) {
IndexQuery indexQuery = new IndexQueryBuilder()
.withId(entityModel.getId())
.withObject(entityModel)
.build();
return elasticsearchTemplate.index(indexQuery);
}
/**
* 批量插入
*
* @param queries 需要更新的 IndexQuery 集合
*/
private void bulkInsert(List<IndexQuery> queries) {
//实际运用中一般是设置为 500 或者 1000
int len = 1000;
List<List<IndexQuery>> indexQueryLists = DataUtils.splitList(queries, len);
indexQueryLists.stream().forEach(indexQueryList -> {
elasticsearchTemplate.bulkIndex(indexQueryList);
});
elasticsearchTemplate.refresh(getEntityClass());
log.info("bulkInsert completed of " + queries.size());
}
/**
* 创建索引和映射
*/
private Boolean createProductEsIndexAndMappers() {
boolean index = elasticsearchTemplate.createIndex(getEntityClass());
if (!index) {
throw new CommonException(" create entity elasticsearch index error");
}
boolean putMapping = elasticsearchTemplate.putMapping(getEntityClass());
if (!putMapping) {
throw new CommonException(" create entity elasticsearch mappers error");
}
return true;
}
@Override
public Page<T> searchPage(NativeSearchQueryBuilder nativeSearchQueryBuilder, int pageNumber, int pageSize) {
nativeSearchQueryBuilder.withIndices(getIndexName());
nativeSearchQueryBuilder.withTypes(getIndexType());
nativeSearchQueryBuilder.withPageable(this.getPageRequest(pageNumber, pageSize));
return elasticsearchTemplate.queryForPage(nativeSearchQueryBuilder.build(), getEntityClass());
}
@Override
public PageRequest getPageRequest(int pageNumber, int pageSize) {
if (pageNumber <= 0) {
throw new CommonException("The page number parameter cannot be less than or equal to 0");
}
//因为是从0开始计算的 所以此处需要自减1
pageNumber--;
return PageRequest.of(pageNumber, pageSize);
}
@Override
public Page<T> searchPageBySort(List<SortDTO> sortDTOList, NativeSearchQueryBuilder nativeSearchQueryBuilder,
int pageNumber, int pageSize) {
this.joinSortBuilders(sortDTOList, nativeSearchQueryBuilder);
return this.searchPage(nativeSearchQueryBuilder, pageNumber, pageSize);
}
@Override
public List<T> searchList(NativeSearchQueryBuilder nativeSearchQueryBuilder) {
nativeSearchQueryBuilder.withIndices(getIndexName());
nativeSearchQueryBuilder.withTypes(getIndexType());
return elasticsearchTemplate.queryForList(nativeSearchQueryBuilder.build(), getEntityClass());
}
@Override
public List<T> searchListBySort(List<SortDTO> sortDTOList, NativeSearchQueryBuilder nativeSearchQueryBuilder) {
this.joinSortBuilders(sortDTOList, nativeSearchQueryBuilder);
return this.searchList(nativeSearchQueryBuilder);
}
/**
* 拼接多个SortBuilders排序条件
*
* @param sortDTOList 排序条件集合
* @param nativeSearchQueryBuilder 查询条件
*/
private void joinSortBuilders(List<SortDTO> sortDTOList, NativeSearchQueryBuilder nativeSearchQueryBuilder) {
for (SortDTO sortDTO : sortDTOList) {
nativeSearchQueryBuilder.withSort(
SortBuilders.fieldSort(sortDTO.getKey()).order(sortDTO.getIsASC() == true ? SortOrder.ASC : SortOrder.DESC)
);
}
}
@Override
public Aggregations query(NativeSearchQueryBuilder nativeSearchQueryBuilder) {
nativeSearchQueryBuilder.withIndices(getIndexName());
nativeSearchQueryBuilder.withTypes(getIndexType());
return elasticsearchTemplate.query(nativeSearchQueryBuilder.build(), searchResponse -> searchResponse.getAggregations());
}
}
3、基础测试类:BaseProductEsTest
package com.xinghua.elasticsearchservice;
import com.xinghua.elasticsearchservice.model.ProductEsModel;
import com.xinghua.elasticsearchservice.service.IProductEsService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
/**
* @Description 基础功能测试类
* @Author 姚广星
* @Date 2020/3/1 12:55
**/
@RunWith(SpringRunner.class)
@SpringBootTest
class BaseProductEsTest {
@Autowired
private IProductEsService productEsService;
@BeforeEach
void setUp() {
//System.out.println("执行初始化");
}
@AfterEach
void tearDown() {
}
/**
* 获取索引名称
*/
@Test
void getIndexName() {
System.out.println(productEsService.getIndexName());
}
/**
* 获取类型
*/
@Test
void getIndexType() {
System.out.println(productEsService.getIndexType());
}
/**
* 删除索引
*/
@Test
void deleteIndex() {
Boolean aBoolean = productEsService.deleteIndex();
System.out.println("aBoolean = " + aBoolean);
}
/**
* 返回泛型上的Class对象
*/
@Test
void getEntityClass() {
System.out.println(productEsService.getEntityClass());
}
/**
* 创建索引和映射(若原有索引存在则删除重新创建)
*/
@Test
void createEntityEsIndex() {
System.out.println(productEsService.createEntityEsIndex());
}
/**
* 初始化数据
* 1: 删除原有的索引
* 2: 创建索引并且初始化映射
* 3: bulk 批量初始化数据
*/
@Test
void init() {
List<ProductEsModel> productEsModels = new ArrayList<>();
ProductEsModel p1 = new ProductEsModel("1", "小米2青春版手机", 1799.99,
"广东省深圳市", "3", "小米", "小米2青春版手机");
ProductEsModel p2 = new ProductEsModel("2", "小米2经典版手机", 1899.99,
"广东省深圳市", "3", "小米", "小米2经典版手机");
ProductEsModel p3 = new ProductEsModel("3", "小米note手机", 2799.99,
"广东省深圳市", "3", "小米", "小米note手机");
ProductEsModel p4 = new ProductEsModel("4", "华为手机1", 1999.99,
"湖南省", "1", "华为", "华为手机1");
ProductEsModel p5 = new ProductEsModel("5", "华为手机2", 2888.88,
"湖南省", "1", "华为", "华为手机2");
ProductEsModel p6 = new ProductEsModel("6", "vivo 10", 999.99,
"广东省广州市", "2", "vivo", "vivo 10");
ProductEsModel p7 = new ProductEsModel("7", "vivo 20", 1899.99,
"广东省广州市", "2", "vivo", "vivo 20");
productEsModels.add(p1);
productEsModels.add(p2);
productEsModels.add(p3);
productEsModels.add(p4);
productEsModels.add(p5);
productEsModels.add(p6);
productEsModels.add(p7);
productEsService.init(productEsModels);
}
/**
* 新增或修改
*/
@Test
void saveOrUpdate() {
System.out.println(productEsService.saveOrUpdate(new ProductEsModel("8", "9", 1899.99,
"广东省广州市", "2", "vivo", "80")));
}
/**
* 删除
*/
@Test
void delete() {
System.out.println(productEsService.delete("8"));
}
/**
* 获取PageRequest对象
*/
@Test
void getPageRequest() {
System.out.println(productEsService.getPageRequest(1, 10));
}
/**
* 批量新增/更新
*/
@Test
void batchInsertOrUpdate() {
List<ProductEsModel> productEsModels = new ArrayList<>();
ProductEsModel p1 = new ProductEsModel("8", "罗技鼠标", 990.00,
"广东省湛江市", "4", "罗技", "罗技鼠标");
ProductEsModel p2 = new ProductEsModel("9", "双飞燕鼠标", 99.99,
"广东省湛江市", "5", "双飞燕", "双飞燕鼠标");
productEsModels.add(p1);
productEsModels.add(p2);
productEsService.batchInsertOrUpdate(productEsModels);
}
}
4、各种高级查询功能测试类 ProductEsTest
package com.xinghua.elasticsearchservice;
import com.xinghua.elasticsearchservice.common.dto.SortDTO;
import com.xinghua.elasticsearchservice.model.ProductEsModel;
import com.xinghua.elasticsearchservice.service.IProductEsService;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.mockito.internal.invocation.MatchersBinder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* @Description 各种高级查询功能测试类
* @Author 姚广星
* @Date 2020/3/1 12:55
**/
@RunWith(SpringRunner.class)
@SpringBootTest
class ProductEsTest {
@Autowired
private IProductEsService productEsService;
@BeforeEach
void setUp() {
//System.out.println("执行初始化");
}
@AfterEach
void tearDown() {
}
/**
* 分页查询-按照价格降序排列,显示第2页,每页显示3个 (分页+排序)
*/
@Test
void query1() {
List<SortDTO> sortDTOList = new ArrayList<>();
sortDTOList.add(new SortDTO("price", false));
Page<ProductEsModel> productEsModels = productEsService.searchPageBySort(sortDTOList, new NativeSearchQueryBuilder(), 2, 3);
productEsModels.forEach(System.out::println);
}
/**
* 分页查询-按照品牌升序然后价格降序排列,显示第2页,每页显示3个 (分页+多条件排序)
*/
@Test
void query2() {
List<SortDTO> sortDTOList = new ArrayList<>();
sortDTOList.add(new SortDTO("brandId", true));
sortDTOList.add(new SortDTO("price", false));
Page<ProductEsModel> productEsModels = productEsService.searchPageBySort(sortDTOList, new NativeSearchQueryBuilder(), 2, 3);
productEsModels.forEach(System.out::println);
}
/**
* 查询商品标题中符合"小米 手机"的字样的商品 (match 查询)
*/
@Test
void query3() {
NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();
searchQueryBuilder.withQuery(
QueryBuilders.matchQuery("title", "小米 手机")
);
List<ProductEsModel> productEsModels = productEsService.searchList(searchQueryBuilder);
productEsModels.forEach(System.out::println);
}
/**
* 查询所有商品标题中带有 鼠标 或者 价格为 1899.99,并且按照价格降序排序 (逻辑查询 must / should / must_not,相当于and / or / not)
*/
@Test
void query4() {
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
nativeSearchQueryBuilder.withQuery(
QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("title", "鼠标"))
.should(QueryBuilders.termQuery("price", 1899.99))
);
List<SortDTO> sortDTOList = new ArrayList<>();
sortDTOList.add(new SortDTO("price", false));
List<ProductEsModel> productEsModels = productEsService.searchListBySort(sortDTOList, nativeSearchQueryBuilder);
productEsModels.forEach(System.out::println);
}
/**
* 查询商品标题或产地中符合"手机,湛江"的字样的商品 (关键字查询)
*/
@Test
void query5() {
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
nativeSearchQueryBuilder.withQuery(
QueryBuilders.multiMatchQuery("手机,湛江", "origin", "title")
);
List<ProductEsModel> productEsModels = productEsService.searchList(nativeSearchQueryBuilder);
productEsModels.forEach(System.out::println);
}
/**
* 查询商品标题中符合"手机"的商品,并且价格在 1900~2800之间
*/
@Test
void query6() {
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
nativeSearchQueryBuilder.withQuery(
QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("title", "手机"))
.must(QueryBuilders.rangeQuery("price").gte(1900).lte(2800))
);
List<ProductEsModel> productEsModels = productEsService.searchList(nativeSearchQueryBuilder);
productEsModels.forEach(System.out::println);
}
/**
* 分组查询(桶聚合) 按照品牌分组,统计各品牌的平均价格
*/
@Test
void query7() {
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
nativeSearchQueryBuilder.withIndices("product").withTypes("product");
nativeSearchQueryBuilder.addAggregation(
AggregationBuilders.terms("groupByBrandId").field("brandId")
.subAggregation(
AggregationBuilders.avg("avgPrice").field("price")
)
);
Aggregations group = productEsService.query(nativeSearchQueryBuilder);
StringTerms groupByBrandId = group.get("groupByBrandId");
List<StringTerms.Bucket> buckets = groupByBrandId.getBuckets();
for (StringTerms.Bucket b : buckets) {
InternalAvg avgPrice = b.getAggregations().get("avgPrice");
System.out.println(avgPrice.getValue());
}
}
@Test
void query8() {
}
}
结束语:
本篇 Elasticsearch 专栏讲到这里可以宣布结束了,博主之所以整理出这个专栏的主要目的是想给各位同行一个比较系统的 Elasticsearch 认识和上手的机会。最后希望感兴趣的朋友们添加一下本人的微信公众号“愿为最亮星”,二维码在下面。谢谢!!!
更多资讯请扫描以下二维码或关注微信公号“愿为最亮星”,为您提供更深层次的解答。