elasticsearch的基本使用介绍参考地址:
https://blog.csdn.net/wjs040/article/details/111031335
高亮显示的代码方法参见下一篇文章:
https://blog.csdn.net/wjs040/article/details/111321107
springboot整合框架的三步曲
1、引入jar包
注意elasticsearch的版本哦,一定要与elasticsearch服务器对应
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
点进去可看到es的版本号,我的就是如下:
<commonscollections>3.2.1</commonscollections>
<commonslang>2.6</commonslang>
<elasticsearch>6.2.2</elasticsearch>
所以ES服务版本也要选择6.xx版本的
2、无须开启注解,需要在代码中引入或继承、实现接口
3、配置文件
spring:
data:
elasticsearch:
repositories:
enabled: true
cluster-nodes: 192.168.22.108:9300
cluster-name: elasticsearch
properties:
transport:
tcp:
connect_timeout: 120s
代码讲解
一、分词属性字段配置声明
/**
@Document:
indexName –> 索引库的名称,建议以项目的名称命名,就相当于数据库DB
type –> 类型,建议以实体的名称命名Table ,就相当于数据库中的表table
*/
@Document(indexName = "pms", type = "product",shards = 1,replicas = 0)
public class EsProduct implements Serializable {
private static final long serialVersionUID = -1L;
@Id // 主键
private Long id;
@Field(type = FieldType.Keyword) //参见下面的FieldType说明
private String productSn;
private Long brandId;
@Field(type = FieldType.Keyword) //关键词,不允许分词匹配
private String brandName;
private Long productCategoryId;
@Field(type = FieldType.Keyword) //关键词,不允许分词匹配
private String productCategoryName;
private String pic;
@Field(analyzer = "ik_max_word",type = FieldType.Text) //可分词,并声明分词规则ik_max_word
private String name;
@Field(analyzer = "ik_max_word",type = FieldType.Text)
private String subTitle;
@Field(analyzer = "ik_max_word",type = FieldType.Text)
private String keywords;
private BigDecimal price;
@Field(index = FieldIndex.not_analyzed) // 不做全文检索字段
private Integer sale;
private Integer newStatus;
private Integer recommandStatus;
private Integer stock;
private Integer promotionType;
private Integer sort;
@Field(type =FieldType.Nested) //表示嵌套类型
private List<EsProductAttributeValue> attrValueList;
}
public class EsProductAttributeValue implements Serializable {
private static final long serialVersionUID = 1L;
private Long id;
private Long productAttributeId;
//属性值
@Field(type = FieldType.Keyword)
private String value;
//属性参数:0->规格;1->参数
private Integer type;
//属性名称
@Field(type=FieldType.Keyword)
private String name;
}
FieldType类型说明
public enum FieldType {
Text, //字符串类型 可进行分词
Integer,//整型
Long,//长整型
Date,//日期类型
Float,//浮点型
Double,
Boolean,
Object,//对象类型
Auto, //自动
Nested, //嵌套类型
Ip, //特殊类型 IP类型
Attachment, //附件类型
Keyword; //关键词,完全匹配,不分词
private FieldType() {
}
}
@Document注解,@Field注解的其他属性说明,参考文章:
https://blog.csdn.net/topdandan/article/details/81436141
二、继承ElasticsearchRepository接口
继承接口ElasticsearchRepository后,可在此写需要的方法,方法名前缀 findByXxxAndXxx;
也就是findBy后面跟上分词类中的属性名,多个查询参数中间用And或Or分隔。idea开发工具中会有提示可以跟哪些属性名的,然后在该方法的参数中,一一写上findBy后面跟的属性名即可;
此处为什么只继承个接口就可以实现查询功能呢,我个人大致看了一下源码,实现这种功能使用的是
AOP+代理 + 反射,应该还有责任链设计模式。如有不对,可留言指正,谢谢。
public interface EsProductRepository extends ElasticsearchRepository<EsProduct, Long> {
/**
* 搜索查询
*
* @param name 商品名称
* @param subTitle 商品标题
* @param keywords 商品关键字
* @param page 分页信息
* @return
*/
Page<EsProduct> findByNameOrSubTitleOrKeywords(String name, String subTitle, String keywords,Pageable page);
Page<EsProduct> findByNameAndSubTitle(String name,String subTitle,Pageable page);
}
智能提示,如下图:
三、接口实现
对es查询实现代码,一一分析
EsProductService 这个接口定义方法,我就不列出来了,接口中的方法都在这个实现类中
@Service
public class EsProductServiceImpl implements EsProductService {
private static final Logger LOGGER = LoggerFactory.getLogger(EsProductServiceImpl.class);
@Autowired
private EsProductDao productDao;// dao 连接数据库进行查询
@Autowired
private EsProductRepository productRepository;//上面二步骤中的es操作类,相当于es原始操作类的增强版
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;//es 原始操作类
@Override
public int importAll() {
// 获取需要分词的所有数据
List<EsProduct> esProductList = productDao.getAllEsProductList(null);
//保存到分词库中,并进行分词
Iterable<EsProduct> esProductIterable = productRepository.saveAll(esProductList);
Iterator<EsProduct> iterator = esProductIterable.iterator();//记录导入的数据
int result = 0;
while (iterator.hasNext()) {
result++;
iterator.next();
}
return result;
}
@Override
public void delete(Long id) {
//根据id删除es分词库中的数据
productRepository.deleteById(id);
}
@Override
public EsProduct create(Long id) {
EsProduct result = null;
//根据id插入到es分词库中的数据
List<EsProduct> esProductList = productDao.getAllEsProductList(id);
if (esProductList.size() > 0) {
EsProduct esProduct = esProductList.get(0);
result = productRepository.save(esProduct);
}
return result;
}
@Override
public void delete(List<Long> ids) {
if (!CollectionUtils.isEmpty(ids)) {
//批量删除es库中的数据
List<EsProduct> esProductList = new ArrayList<>();
for (Long id : ids) {
EsProduct esProduct = new EsProduct();
esProduct.setId(id);
esProductList.add(esProduct);
}
productRepository.deleteAll(esProductList);
}
}
//简单搜索
@Override
public Page<EsProduct> search(String keyword, Integer pageNum, Integer pageSize) {
//声明分页对象
Pageable pageable = PageRequest.of(pageNum, pageSize);
//根据名称、标题、关键词进行匹配检索,此方法就是前面继承ElasticsearchRepository接口自定义的
//方法
return productRepository.findByNameOrSubTitleOrKeywords(keyword, keyword, keyword, pageable);
}
//高级搜索
/**
讲解之前先说明一下主要查询流程,
1、创建查询条件构建器
2、创建各种类型的查询条件,如基本查询条件、布尔查询条件、过滤查询条件、分页、排序等
即:nativeSearchQueryBuilder.withQuery("基本或布尔").withFilter("基本或布尔")
.withSort("排序条件").withPageable("分页");
3、构建查询条件NativeSearchQuery searchQuery= nativeSearchQueryBuilder.build();
4、查询productRepository.search(searchQuery);
所有的查询都是按照该流程的,主要有2个类QueryBuilder和SearchQuery,只不过在使用的时候大多使用的是某些类的子类
*/
@Override
public Page<EsProduct> search(String keyword, Long brandId, Long productCategoryId, Integer pageNum, Integer pageSize,Integer sort) {
//分页
Pageable pageable = PageRequest.of(pageNum, pageSize);
// 查询构建器
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
//将分页条件放入查询构建器中
nativeSearchQueryBuilder.withPageable(pageable);
//过滤
if (brandId != null || productCategoryId != null) {
//创建布尔过滤条件
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (brandId != null) {
//不分词,完全匹配的条件 must必须都相等,term精确查找
boolQueryBuilder.must(QueryBuilders.termQuery("brandId", brandId));
}
if (productCategoryId != null) {
//不分词,完全匹配的条件
boolQueryBuilder.must(QueryBuilders.termQuery("productCategoryId", productCategoryId));
}
//将布尔过滤条件放入构建器中,需注此处的过滤,过滤出来的数据时满足条件的数据,
//需输出过滤出来的结果,下面的聚合查询的过滤是排除掉过滤的结果数据
nativeSearchQueryBuilder.withFilter(boolQueryBuilder);
}
//搜索
if (StringUtils.isEmpty(keyword)) {
//基本类型的查询,此处是查询所有,按照从上到下的执行顺序的话,此处的结果数据是经过
//上面过滤的数据
nativeSearchQueryBuilder.withQuery(QueryBuilders.matchAllQuery());
} else {
//高级过滤条件,权重得分过滤器,权重越高则优先按此条件查询过滤
List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders = new ArrayList<>();
//按照name匹配,权重值设置为10
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("name", keyword),
ScoreFunctionBuilders.weightFactorFunction(10)));
//按照subTitle匹配,权重值为5
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("subTitle", keyword),
ScoreFunctionBuilders.weightFactorFunction(5)));
//按照keywords匹配,权重值为2
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("keywords", keyword),
ScoreFunctionBuilders.weightFactorFunction(2)));
//由于下面的构造器是传入的数据,所以此处声明一个数据,将过滤条件集合转为数组
FunctionScoreQueryBuilder.FilterFunctionBuilder[] builders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[filterFunctionBuilders.size()];
//将集合数据放入数组中
filterFunctionBuilders.toArray(builders);
//创建得分查询构造器,并将得分过滤查询条件放入构造器中
FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(builders)
.scoreMode(FunctionScoreQuery.ScoreMode.SUM)//说明此查询条件的各个参数过滤后的数据是按照怎样的形式输出数据。一般使用默认即可
/**有这几种模式
FIRST,使用首个函数
AVG, 取平均值
MAX, old_score 和 加强score 取较大值,new_score = max(old_score, 加强score)
SUM, new_score = old_score + 加强score
MIN, old_score 和 加强score 取较小值,new_score = min(old_score, 加强score)
MULTIPLY;(默认)new_score = old_score * 加强score
*/
.setMinScore(2);//设置查询结果的最小得分,如果得分不大于2,则不输出此结果
//将此查询条件放入到查询构造器中
nativeSearchQueryBuilder.withQuery(functionScoreQueryBuilder);
}
//排序 下面是按照不同的需求进行排序,就没什么可讲的了
if(sort==1){
//按新品从新到旧
nativeSearchQueryBuilder.withSort(SortBuilders.fieldSort("id").order(SortOrder.DESC));
}else if(sort==2){
//按销量从高到低
nativeSearchQueryBuilder.withSort(SortBuilders.fieldSort("sale").order(SortOrder.DESC));
}else if(sort==3){
//按价格从低到高
nativeSearchQueryBuilder.withSort(SortBuilders.fieldSort("price").order(SortOrder.ASC));
}else if(sort==4){
//按价格从高到低
nativeSearchQueryBuilder.withSort(SortBuilders.fieldSort("price").order(SortOrder.DESC));
}else{
//按相关度
nativeSearchQueryBuilder.withSort(SortBuilders.scoreSort().order(SortOrder.DESC));
}
//将排序条件放入查询条件构造器中
nativeSearchQueryBuilder.withSort(SortBuilders.scoreSort().order(SortOrder.DESC));
//构建查询条件
NativeSearchQuery searchQuery = nativeSearchQueryBuilder.build();
LOGGER.info("DSL:{}", searchQuery.getQuery().toString());
//按照条件进行查询
return productRepository.search(searchQuery);
}
//根据给出的商品id,推荐类似的商品
@Override
public Page<EsProduct> recommend(Long id, Integer pageNum, Integer pageSize) {
Pageable pageable = PageRequest.of(pageNum, pageSize);
List<EsProduct> esProductList = productDao.getAllEsProductList(id);
if (esProductList.size() > 0) {
//根据id从数据库中查询出一条数据
EsProduct esProduct = esProductList.get(0);
String keyword = esProduct.getName();
Long brandId = esProduct.getBrandId();
Long productCategoryId = esProduct.getProductCategoryId();
//根据商品标题、品牌、分类进行搜索
List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders = new ArrayList<>();
//可以根据下面这些条件,获取类似的商品,下面也是通过得分权重过滤器进行过滤查询的,同上
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("name", keyword),
ScoreFunctionBuilders.weightFactorFunction(8)));
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("subTitle", keyword),
ScoreFunctionBuilders.weightFactorFunction(2)));
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("keywords", keyword),
ScoreFunctionBuilders.weightFactorFunction(2)));
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("brandId", brandId),
ScoreFunctionBuilders.weightFactorFunction(10)));
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("productCategoryId", productCategoryId),
ScoreFunctionBuilders.weightFactorFunction(6)));
FunctionScoreQueryBuilder.FilterFunctionBuilder[] builders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[filterFunctionBuilders.size()];
filterFunctionBuilders.toArray(builders);
FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(builders)
.scoreMode(FunctionScoreQuery.ScoreMode.SUM)
.setMinScore(2);
NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
builder.withQuery(functionScoreQueryBuilder);
builder.withPageable(pageable);
NativeSearchQuery searchQuery = builder.build();
LOGGER.info("DSL:{}", searchQuery.getQuery().toString());
return productRepository.search(searchQuery);
}
return new PageImpl<>(null);
}
//高级查询,关联查询,聚合查询
@Override
public EsProductRelatedInfo searchRelatedInfo(String keyword) {
//创建查询构造器
NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
//搜索条件
if(StringUtils.isEmpty(keyword)){
//若关键词为空,则查询所有的信息
builder.withQuery(QueryBuilders.matchAllQuery());
}else{
/**
多字段匹配查询
*/ builder.withQuery(QueryBuilders.multiMatchQuery(keyword,"name","subTitle","keywords"));
}
//上面是查询数据的条件。下面是处理查询出来的数据的聚合参数的输出情况,即需要聚合哪些数据以及参数命名
//聚合搜索品牌名称
builder.addAggregation(AggregationBuilders.terms("brandNames").field("brandName"));
//集合搜索分类名称
builder.addAggregation(AggregationBuilders.terms("productCategoryNames").field("productCategoryName"));
//聚合搜索商品属性,去除type=1的属性
AbstractAggregationBuilder aggregationBuilder =
//此处的属性是嵌套所以用nested,allAttrValues这个参数是自定义命名,相当于集合的名称;
//attrValueList这个参数是在es分词库中的属性名称,聚合后的类InternalNested
AggregationBuilders.nested("allAttrValues","attrValueList")
.subAggregation(
/**
此处是过滤type=1的属性的数据后,聚合后的名称productAttrs,是allAttrValues的一个子集
聚合后的类InternalFilter
*/ AggregationBuilders.filter("productAttrs",QueryBuilders.termQuery("attrValueList.type",1))
.subAggregation(
/**
此处是在productAttrs中聚合数据,根据attrValueList.productAttributeId这个属性进行聚合数据,聚合后数据的名是attrIds,属于productAttrs子集
聚合后的类LongTerms,桶里面的数据 LongTerms.Bucket
*/ AggregationBuilders.terms("attrIds").field("attrValueList.productAttributeId")
/** 此处是在attrIds里面聚合,按照attrValueList.value聚合,聚合后的名称为attrValues 属于attrIds子集
StringTerms.Bucket
*/ .subAggregation(AggregationBuilders.terms("attrValues").field("attrValueList.value"))
.subAggregation(AggregationBuilders.terms("attrNames").field("attrValueList.name"))
)
);
/**
可能大家看到上面介绍的还有点懵。大家可以将allAttrValues,productAttrs,attrIds这三个理解为一个桶,这三个里面都有子集,即attrIds桶在productAttrs桶里,productAttrs桶在allAttrValues桶里;每个聚合桶里面,放的都是一个个文档数据,再获取的时候相当于一个文档数据就是一个小桶。
allAttrValues{
productAttrs{
attrIds{
{数据},attrValues{数据},attrNames{数据}
}
}
}
而attrIds桶里又有2个元素attrValues,attrNames
*/
/** 将聚合构建的数据参数放入查询构造器中*/
builder.addAggregation(aggregationBuilder);
NativeSearchQuery searchQuery = builder.build();
//然后使用原始类调用查询
return elasticsearchTemplate.query(searchQuery, response -> {
LOGGER.info("DSL:{}",searchQuery.getQuery().toString());
return convertProductRelatedInfo(response);
});
}
/**
* 将返回结果转换为对象
*/
private EsProductRelatedInfo convertProductRelatedInfo(SearchResponse response) {
EsProductRelatedInfo productRelatedInfo = new EsProductRelatedInfo();
Map<String, Aggregation> aggregationMap = response.getAggregations().getAsMap();
//设置品牌
Aggregation brandNames = aggregationMap.get("brandNames");//根据聚合名称获取聚合类
List<String> brandNameList = new ArrayList<>();
//获取该聚合类中的桶以及桶内的数据,((Terms) brandNames).getBuckets()此处的Terms也可以使用子类StringTerms,因为是字符串类型的
/** 获取的值得例子 aggregationMap.get("brandNames")
{"brandNames":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"小米","doc_count":4}]}}
此处的brandNames就是上面进行查询时自定义的参数名称相当于一个桶。该桶里有很多小桶即数据;
key:就是查询时定义的field参数的值,doc_count:表示查询出来的数据中有多少个文档包含key值
*/
for(int i = 0; i<((Terms) brandNames).getBuckets().size(); i++){
brandNameList.add(((Terms) brandNames).getBuckets().get(i).getKeyAsString());
}
productRelatedInfo.setBrandNames(brandNameList);
//设置分类 此处的数据处理同上 brandNames
/**
{"productCategoryNames":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,"buckets":[{"key":"手机数码","doc_count":2},{"key":"手机通讯","doc_count":2}]}}
*/
Aggregation productCategoryNames = aggregationMap.get("productCategoryNames");
List<String> productCategoryNameList = new ArrayList<>();
for(int i=0;i<((Terms) productCategoryNames).getBuckets().size();i++){
productCategoryNameList.add(((Terms) productCategoryNames).getBuckets().get(i).getKeyAsString());
}
productRelatedInfo.setProductCategoryNames(productCategoryNameList);
//设置参数 allAttrValues这个参数名就是nested里面命名的
//数据参考本文最下方
/**
(LongTerms) ((InternalFilter) ((InternalNested) productAttrs
这些转换类型就是在构造参数时一层层嵌套的,依次是nested,filter,attrIds(Long型),与上面定义时的一一对应
*/
Aggregation productAttrs = aggregationMap.get("allAttrValues");
List<LongTerms.Bucket> attrIds = ((LongTerms) ((InternalFilter) ((InternalNested) productAttrs).getProperty("productAttrs")).getProperty("attrIds")).getBuckets();
List<EsProductRelatedInfo.ProductAttr> attrList = new ArrayList<>();
for (Terms.Bucket attrId : attrIds) {
EsProductRelatedInfo.ProductAttr attr = new EsProductRelatedInfo.ProductAttr();
attr.setAttrId((Long) attrId.getKey());
List<String> attrValueList = new ArrayList<>();
List<StringTerms.Bucket> attrValues = ((StringTerms) attrId.getAggregations().get("attrValues")).getBuckets();
List<StringTerms.Bucket> attrNames = ((StringTerms) attrId.getAggregations().get("attrNames")).getBuckets();
for (Terms.Bucket attrValue : attrValues) {
attrValueList.add(attrValue.getKeyAsString());
}
attr.setAttrValues(attrValueList);
if(!CollectionUtils.isEmpty(attrNames)){
String attrName = attrNames.get(0).getKeyAsString();
attr.setAttrName(attrName);
}
attrList.add(attr);
}
productRelatedInfo.setProductAttrs(attrList);
return productRelatedInfo;
}
}
allAttrValues 这个桶里面的数据,下面的输出值内容即allAttrValues(productAttrs(attrIds))
这个里面的值数据如下:
{
"allAttrValues": { //nested 名称
"doc_count": 10,
"productAttrs": { //filter过滤的名称
"doc_count": 8,
"attrIds": { //ids 名称
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{ //ids的桶
"key": 45, //id 属性id
"doc_count": 2,
"attrValues": { //该属性下的参数值,规格
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "5.0",
"doc_count": 1
}, {
"key": "5.8",
"doc_count": 1
}]
},
"attrNames": { //属性名称
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "屏幕尺寸",
"doc_count": 2
}]
}
}, {
"key": 46,
"doc_count": 2,
"attrValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "4G",
"doc_count": 2
}]
},
"attrNames": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "网络",
"doc_count": 2
}]
}
}, {
"key": 47,
"doc_count": 2,
"attrValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "Android",
"doc_count": 2
}]
},
"attrNames": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "系统",
"doc_count": 2
}]
}
}, {
"key": 48,
"doc_count": 2,
"attrValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "2800ml",
"doc_count": 1
}, {
"key": "3000ml",
"doc_count": 1
}]
},
"attrNames": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [{
"key": "电池容量",
"doc_count": 2
}]
}
}]
}
}
}
}