0. 高级篇-课件-06、ElasticSearch.pdf
Welcome to Elastic Docs | Elastic
是什么：
Elastic 的底层是开源库 Lucene。
Elasticsearch 会以 JSON 文档的形式存储数据。
Elasticsearch 使用的是一种名为倒排索引的数据结构，这一结构的设计可以允许十分快速地进行全文本搜索。倒排索引会列出在所有文档中出现的每个特有词汇，并且可以找到包含每个词汇的全部文档。
弊端（传统关系型数据库的缺点，为什么使用es）：
1、 对于传统的关系性数据库对于关键词的查询，只能逐字逐行的匹配，性能非常差。
2、匹配方式不合理，比如搜索“小密手机” ，如果用like进行匹配， 根本匹配不到。但是考虑使用者的用户体验的话，除了完全匹配的记录，还应该显示一部分近似匹配的记录，至少应该匹配到“手机”。
es的使用场景：
- 为用户提供按关键字查询的全文搜索功能。
- 著名的ELK框架(ElasticSearch,Logstash,Kibana)，实现企业海量日志的处理分析的解决方案。大数据领域的重要一份子。
1. 上架（后台管理系统）
商品有上架，下架，新建三个状态。
商品全部信息本身存储在mysql，es中保存sku，spu等关键属性信息组成的json文档，查到id后再关联mysql查出所有商品属性。
2. 检索（前台商城系统）
两种查询方式：分类查询和全文检索
2.0 dsl
product_mapping.json
PUT gulimall_product
{
"mappings": {
"properties": {
"attrs": {
"type": "nested",
"properties": {
"attrId": {
"type": "long"
},
"attrName": {
"type": "keyword"
},
"attrValue": {
"type": "keyword"
}
}
},
"brandId": {
"type": "long"
},
"brandImg": {
"type": "keyword"
},
"brandName": {
"type": "keyword"
},
"catalogId": {
"type": "long"
},
"catalogName": {
"type": "keyword"
},
"hasStock": {
"type": "boolean"
},
"hotScore": {
"type": "long"
},
"saleCount": {
"type": "long"
},
"skuId": {
"type": "long"
},
"skuImg": {
"type": "keyword"
},
"skuPrice": {
"type": "keyword"
},
"skuTitle": {
"type": "text",
"analyzer": "ik_smart"
},
"spuId": {
"type": "keyword"
}
}
}
}dsl.json
# 模糊匹配，过滤（按照属性，分类，品牌，价格区间，库存），排序，分页，高亮，聚合分析
# 如果是嵌入式的属性；查询，聚合，分析都应该是嵌入式的。
GET gulimall_product/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"skuTitle": "华为"
}
}
],
"filter": [
{
"term": {
"catalogId": "225"
}
},
{
"terms": {
"brandId": [
"1",
"2",
"9"
]
}
},
{
"nested": {
"path": "attrs",
"query": {
"bool": {
"must": [
{
"term": {
"attrs.attrId": {
"value": "15"
}
}
},
{
"terms": {
"attrs.attrValue": [
"海思（Hisilicon）",
"以官网信息为准"
]
}
}
]
}
}
}
},
{
"term": {
"hasStock": {
"value": "true"
}
}
},
{
"range": {
"skuPrice": {
"gte": 0,
"lte": 6000
}
}
}
]
}
},
"sort": [
{
"skuPrice": {
"order": "desc"
}
}
],
"from": 0,
"size": 1,
"highlight": {
"fields": {
"skuTitle": {}
},
"pre_tags": "<b style='color:red'>",
"post_tags": "</b>"
},
"aggs": {
"brand_agg": {
"terms": {
"field": "brandId",
"size": 10
},
"aggs": {
"brand_name_agg": {
"terms": {
"field": "brandName",
"size": 10
}
},
"brand_img_agg": {
"terms": {
"field": "brandImg",
"size": 10
}
}
}
},
"catalog_agg": {
"terms": {
"field": "catalogId",
"size": 10
},
"aggs": {
"catalog_name_agg": {
"terms": {
"field": "catalogName",
"size": 10
}
}
}
},
"attr_agg": {
"nested": {
"path": "attrs"
},
"aggs": {
"attr_id_agg": {
"terms": {
"field": "attrs.attrId",
"size": 10
},
"aggs": {
"attr_name_agg": {
"terms": {
"field": "attrs.attrName",
"size": 1
}
},
"attr_value_agg": {
"terms": {
"field": "attrs.attrValue",
"size": 10
}
}
}
}
}
}
}
}2.1 配置：springboot整合es
pom.xml
<!-- low-level client -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.4.2</version>
</dependency>
<!-- high-level client -->
<!-- Elasticsearch-Rest-Client： 官方 RestClient， 封装了 ES 操作， API 层次分明， 上手简单 -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.4.2</version>
</dependency>
<!-- 使用low-level client不需要，但是high-level client需要依赖-->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.4.2</version>
</dependency>2.2 Constant
EsConstant.java
package com.atguigu.gulimall.search.constant;
public class EsConstant {
//在es中的索引
public static final String PRODUCT_INDEX = "gulimall_product";
public static final Integer PRODUCT_PAGESIZE = 16;
}2.3 Config
GulimallElasticSearchConfig.java
package com.atguigu.gulimall.search.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class GulimallElasticSearchConfig {
public static final RequestOptions COMMON_OPTIONS;
static {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
// builder.addHeader("Authorization", "Bearer " + TOKEN);
// builder.setHttpAsyncResponseConsumerFactory(
// new HttpAsyncResponseConsumerFactory
// .HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
COMMON_OPTIONS = builder.build();
}
@Bean
public RestHighLevelClient esRestClient(){
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(new HttpHost("192.168.56.10", 9200, "http")));
return client;
}
}
2.4 VO
SearchParam.java
package com.atguigu.gulimall.search.vo;
import lombok.Data;
import java.util.List;
@Data
public class SearchParam {
/**
* 页面传递过来的全文匹配关键字
*/
private String keyword;
/**
* 品牌id,可以多选
*/
private List<Long> brandId;
/**
* 三级分类id
*/
private Long catalog3Id;
/**
* 排序条件：sort=price/salecount/hotscore_desc/asc
*/
private String sort;
/**
* 是否显示有货
*/
private Integer hasStock;
/**
* 价格区间查询
*/
private String skuPrice;
/**
* 按照属性进行筛选
*/
private List<String> attrs;
/**
* 页码
*/
private Integer pageNum = 1;
/**
* 原生的所有查询条件
*/
private String _queryString;
}
SearchResult.java
package com.atguigu.gulimall.search.vo;
import com.atguigu.common.to.es.SkuEsModel;
import lombok.Data;
import java.util.List;
@Data
public class SearchResult {
/**
* 查询到的所有商品信息
*/
private List<SkuEsModel> product;
/**
* 当前页码
*/
private Integer pageNum;
/**
* 总记录数
*/
private Long total;
/**
* 总页码
*/
private Integer totalPages;
private List<Integer> pageNavs;
/**
* 当前查询到的结果，所有涉及到的品牌
*/
private List<BrandVo> brands;
/**
* 当前查询到的结果，所有涉及到的所有属性
*/
private List<AttrVo> attrs;
/**
* 当前查询到的结果，所有涉及到的所有分类
*/
private List<CatalogVo> catalogs;
//===========================以上是返回给页面的所有信息============================//
/* 面包屑导航数据 */
private List<NavVo> navs;
@Data
public static class NavVo {
private String navName;
private String navValue;
private String link;
}
@Data
public static class BrandVo {
private Long brandId;
private String brandName;
private String brandImg;
}
@Data
public static class AttrVo {
private Long attrId;
private String attrName;
private List<String> attrValue;
}
@Data
public static class CatalogVo {
private Long catalogId;
private String catalogName;
}
}
2.5 Controller
SearchController.java
package com.atguigu.gulimall.search.controller;
import com.atguigu.gulimall.search.service.MallSearchService;
import com.atguigu.gulimall.search.vo.SearchParam;
import com.atguigu.gulimall.search.vo.SearchResult;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import javax.servlet.http.HttpServletRequest;
@Controller
public class SearchController {
@Autowired
MallSearchService mallSearchService;
@GetMapping("/list.html")
public String listPage(SearchParam param, Model model, HttpServletRequest request){
param.set_queryString(request.getQueryString());
SearchResult result = mallSearchService.search(param);
model.addAttribute("result", result);
return "list";
}
}
2.6 Service
- 请求带来的参数是SearchParam
- 传给es的参数是SearchRequest
- es返回结果是SearchResponse
- 把结果封装为SearchResult
MallSearchService.java
package com.atguigu.gulimall.search.service;
import com.atguigu.gulimall.search.vo.SearchParam;
import com.atguigu.gulimall.search.vo.SearchResult;
public interface MallSearchService {
SearchResult search(SearchParam param);
}
MallSearchServiceImpl.java
buildSearchRequest：SearchSourceBuilder(): DSL -> java，QueryBuilders(): es中提供的一个查询接口, SearchRequest()
buildSearchResult：response.getHits() 查询所有记录，searchResult.set() 设置返回结果
package com.atguigu.gulimall.search.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.atguigu.common.to.es.SkuEsModel;
import com.atguigu.common.utils.R;
import com.atguigu.gulimall.search.config.GulimallElasticSearchConfig;
import com.atguigu.gulimall.search.constant.EsConstant;
import com.atguigu.gulimall.search.feign.ProductFeignService;
import com.atguigu.gulimall.search.service.MallSearchService;
import com.atguigu.gulimall.search.vo.AttrResponseVo;
import com.atguigu.gulimall.search.vo.SearchParam;
import com.atguigu.gulimall.search.vo.SearchResult;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Service
public class MallSearchServiceImpl implements MallSearchService {
@Autowired
private RestHighLevelClient client;
@Resource
private ProductFeignService productFeignService;
@Override
public SearchResult search(SearchParam param) {
//1、动态构建出查询需要的DSL语句
SearchResult result = null;
//1、准备检索请求(DSL转java)
SearchRequest searchRequest = buildSearchRequest(param);
try {
//2、执行检索请求
SearchResponse response = client.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
//3、分析响应数据，封装成我们需要的格式
result = buildSearchResult(response,param);
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
// 得到检索到的商品, 还有聚合信息
private SearchResult buildSearchResult(SearchResponse response,SearchParam param) {
SearchResult result = new SearchResult();
//1、返回的所有查询到的商品
SearchHits hits = response.getHits();
List<SkuEsModel> esModels = new ArrayList<>();
//遍历所有商品信息
if (hits.getHits() != null && hits.getHits().length > 0) {
for (SearchHit hit : hits.getHits()) {
String sourceAsString = hit.getSourceAsString();
SkuEsModel esModel = JSON.parseObject(sourceAsString, SkuEsModel.class);
//判断是否按关键字检索，若是就显示高亮，否则不显示
if (!StringUtils.isEmpty(param.getKeyword())) {
//拿到高亮信息显示标题
HighlightField skuTitle = hit.getHighlightFields().get("skuTitle");
String skuTitleValue = skuTitle.getFragments()[0].string();
esModel.setSkuTitle(skuTitleValue);
}
esModels.add(esModel);
}
}
result.setProduct(esModels);
//2、当前商品涉及到的所有属性信息
List<SearchResult.AttrVo> attrVos = new ArrayList<>();
//获取属性信息的聚合
ParsedNested attrsAgg = response.getAggregations().get("attr_agg");
ParsedLongTerms attrIdAgg = attrsAgg.getAggregations().get("attr_id_agg");
for (Terms.Bucket bucket : attrIdAgg.getBuckets()) {
SearchResult.AttrVo attrVo = new SearchResult.AttrVo();
//1、得到属性的id
long attrId = bucket.getKeyAsNumber().longValue();
attrVo.setAttrId(attrId);
//2、得到属性的名字
ParsedStringTerms attrNameAgg = bucket.getAggregations().get("attr_name_agg");
String attrName = attrNameAgg.getBuckets().get(0).getKeyAsString();
attrVo.setAttrName(attrName);
//3、得到属性的所有值
ParsedStringTerms attrValueAgg = bucket.getAggregations().get("attr_value_agg");
List<String> attrValues = attrValueAgg.getBuckets().stream().map(item -> item.getKeyAsString()).collect(Collectors.toList());
attrVo.setAttrValue(attrValues);
attrVos.add(attrVo);
}
result.setAttrs(attrVos);
//3、当前商品涉及到的所有品牌信息
List<SearchResult.BrandVo> brandVos = new ArrayList<>();
//获取到品牌的聚合
ParsedLongTerms brandAgg = response.getAggregations().get("brand_agg");
for (Terms.Bucket bucket : brandAgg.getBuckets()) {
SearchResult.BrandVo brandVo = new SearchResult.BrandVo();
//1、得到品牌的id
long brandId = bucket.getKeyAsNumber().longValue();
brandVo.setBrandId(brandId);
//2、得到品牌的名字
ParsedStringTerms brandNameAgg = bucket.getAggregations().get("brand_name_agg");
String brandName = brandNameAgg.getBuckets().get(0).getKeyAsString();
brandVo.setBrandName(brandName);
//3、得到品牌的图片
ParsedStringTerms brandImgAgg = bucket.getAggregations().get("brand_img_agg");
String brandImg = brandImgAgg.getBuckets().get(0).getKeyAsString();
brandVo.setBrandImg(brandImg);
brandVos.add(brandVo);
}
result.setBrands(brandVos);
//4、当前商品涉及到的所有分类信息
//获取到分类的聚合
List<SearchResult.CatalogVo> catalogVos = new ArrayList<>();
ParsedLongTerms catalogAgg = response.getAggregations().get("catalog_agg");
for (Terms.Bucket bucket : catalogAgg.getBuckets()) {
SearchResult.CatalogVo catalogVo = new SearchResult.CatalogVo();
//得到分类id
String keyAsString = bucket.getKeyAsString();
catalogVo.setCatalogId(Long.parseLong(keyAsString));
//得到分类名
ParsedStringTerms catalogNameAgg = bucket.getAggregations().get("catalog_name_agg");
String catalogName = catalogNameAgg.getBuckets().get(0).getKeyAsString();
catalogVo.setCatalogName(catalogName);
catalogVos.add(catalogVo);
}
result.setCatalogs(catalogVos);
//===============以上可以从聚合信息中获取====================//
//5、分页信息-页码
result.setPageNum(param.getPageNum());
//5、1分页信息、总记录数
long total = hits.getTotalHits().value;
result.setTotal(total);
//5、2分页信息-总页码-计算
int totalPages = (int)total % EsConstant.PRODUCT_PAGESIZE == 0 ?
(int)total / EsConstant.PRODUCT_PAGESIZE : ((int)total / EsConstant.PRODUCT_PAGESIZE + 1);
result.setTotalPages(totalPages);
List<Integer> pageNavs = new ArrayList<>();
for (int i = 1; i <= totalPages; i++) {
pageNavs.add(i);
}
result.setPageNavs(pageNavs);
//6、构建面包屑导航
if (param.getAttrs() != null && param.getAttrs().size() > 0) {
List<SearchResult.NavVo> collect = param.getAttrs().stream().map(attr -> {
//1、分析每一个attrs传过来的参数值
SearchResult.NavVo navVo = new SearchResult.NavVo();
String[] s = attr.split("_");
navVo.setNavValue(s[1]);
R r = productFeignService.attrInfo(Long.parseLong(s[0]));
if (r.getCode() == 0) {
AttrResponseVo data = r.getData("attr", new TypeReference<AttrResponseVo>() {
});
navVo.setNavName(data.getAttrName());
} else {
navVo.setNavName(s[0]);
}
//2、取消了这个面包屑以后，我们要跳转到哪个地方，将请求的地址url里面的当前置空
//拿到所有的查询条件，去掉当前
String encode = null;
try {
encode = URLEncoder.encode(attr,"UTF-8");
encode.replace("+","%20"); //浏览器对空格的编码和Java不一样，差异化处理
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
String replace = param.get_queryString().replace("&attrs=" + attr, "");
navVo.setLink("http://search.gulimall.com/list.html?" + replace);
return navVo;
}).collect(Collectors.toList());
result.setNavs(collect);
}
return result;
}
// 准备检索请求(DSL转java)
private SearchRequest buildSearchRequest(SearchParam param) {
SearchSourceBuilder sourceBuilder= new SearchSourceBuilder(); // 构建DSL语句
//查询：模糊匹配，过滤
//1.构建bool - query
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
sourceBuilder.query(boolQuery);
//1.1 must模糊匹配
if(!StringUtils.isEmpty(param.getKeyword())){
boolQuery.must(QueryBuilders.matchQuery("skuTitle",param.getKeyword()));
}
//1.2 bool - filter - 按照三级分类id查询
if(param.getCatalog3Id()!=null){
boolQuery.filter(QueryBuilders.termQuery("catelogId",param.getCatalog3Id()));
}
//1.3 bool - filter - 按照品牌id查询
if(param.getBrandId()!=null && param.getBrandId().size()>0){
boolQuery.filter(QueryBuilders.termsQuery("brandId",param.getBrandId()));
}
//1.4 bool - filter - 按照所有指定属性查询
if(param.getAttrs()!=null && param.getAttrs().size()>0){
for(String attrStr : param.getAttrs()){
BoolQueryBuilder nestedboolQuery = QueryBuilders.boolQuery();
String[] s = attrStr.split("_");
String attrId = s[0];
String[] attrValues = s[1].split(":");
nestedboolQuery.must(QueryBuilders.termQuery("attrs.attrId",attrId));
nestedboolQuery.must(QueryBuilders.termQuery("attrs.attrValue",attrValues));
NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("attrs", nestedboolQuery, ScoreMode.None);
boolQuery.filter(nestedQuery);
}
}
//1.5 bool - filter - 按照是否有库存查询
if(param.getHasStock() != null){
boolQuery.filter(QueryBuilders.termQuery("hasStock",param.getHasStock()==1));
}
//1.6 bool - filter - 按照价格区间查询
if(!StringUtils.isEmpty(param.getSkuPrice())){
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("skuPrice");
String[] s = param.getSkuPrice().split("_");
if(s.length == 2){
rangeQuery.gte(s[0]).lte(s[1]);
}else if(s.length == 1){
if(param.getSkuPrice().startsWith("_")){
rangeQuery.lte(s[0]);
}
if(param.getSkuPrice().endsWith("_")){
rangeQuery.gte(s[0]);
}
}
}
//1.7 把以前的所有条件都拿来封装
sourceBuilder.query(boolQuery);
//2.排序，分类，高亮
//2.1 排序
if(!StringUtils.isEmpty(param.getSort())){
String sort = param.getSort();
String[] s = sort.split("_");
SortOrder order = s[1].equalsIgnoreCase("asc")?SortOrder.ASC:SortOrder.DESC;
sourceBuilder.sort(s[0], SortOrder.ASC);
}
//2.2 分页
sourceBuilder.from((param.getPageNum()-1)*EsConstant.PRODUCT_PAGESIZE);
sourceBuilder.size(EsConstant.PRODUCT_PAGESIZE);
//2.3 高亮
if(StringUtils.isEmpty(param.getKeyword())){
HighlightBuilder builder = new HighlightBuilder();
builder.field("skuTitle");
builder.preTags("b style='color:red'");
builder.postTags("</b>");
sourceBuilder.highlighter(builder);
}
//聚合分析
//1 品牌聚合
TermsAggregationBuilder brand_agg = AggregationBuilders.terms("brand_agg");
brand_agg.field("brandId").size(50);
//品牌聚合子聚合
brand_agg.subAggregation(AggregationBuilders.terms("brand_name").field("brandName").size(1));
brand_agg.subAggregation(AggregationBuilders.terms("brand_img_agg").field("brandImg").size(1));
sourceBuilder.aggregation(brand_agg);
//2 分类聚合
TermsAggregationBuilder catalog_agg = AggregationBuilders.terms("catalog_agg").field("catalogId").size(20);
catalog_agg.subAggregation(AggregationBuilders.terms("catalog_name_agg").field("catalogName").size(1));
sourceBuilder.aggregation(catalog_agg);
//3 属性聚合
NestedAggregationBuilder attr_agg = AggregationBuilders.nested("attr_agg", "attrs");
//聚合attrId
TermsAggregationBuilder attr_id_agg = AggregationBuilders.terms("attr_id_agg").field("attrs.attrId");
//聚合分析当前attrId对应的名字
attr_id_agg.subAggregation(AggregationBuilders.terms("attr_name_agg").field("attrs.attrName").size(1));
//聚合分析当前attrValue对应的可能的属性值
attr_id_agg.subAggregation(AggregationBuilders.terms("attr_name_agg").field("attrs.attrName").size(1));
attr_agg.subAggregation(attr_id_agg);
sourceBuilder.aggregation(attr_agg);
//打印DSL语句
String s = sourceBuilder.toString();
System.out.println(s);
SearchRequest searchRequest = new SearchRequest(new String[]{EsConstant.PRODUCT_INDEX}, sourceBuilder);
return searchRequest;
}
}
999
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
