六,搜索(数据导入es)
6.1搭建搜索工程![在这里插入图片描述](https://img-blog.csdnimg.cn/8aeb56634102470a8a46bc8d5a55d9ed.png)
legou-search/pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>legou-parent</artifactId>
<groupId>com.lxs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>legou-search</artifactId>
<packaging>pom</packaging>
<modules>
<module>legou-search-instance</module>
<module>legou-search-service</module>
</modules>
</project>
legou-search/legou-search-instance/pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>legou-search</artifactId>
<groupId>com.lxs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>legou-search-instance</artifactId>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
<!--elasticsearch-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<!--商品微服务-->
<dependency>
<groupId>com.lxs</groupId>
<artifactId>legou-item-instance</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.core.Starter</mainClass>
<layout>ZIP</layout>
<classifier>exec</classifier>
<includeSystemScope>true</includeSystemScope>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
legou-search/legou-search-service/pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>legou-search</artifactId>
<groupId>com.lxs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>legou-search-service</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-hystrix</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
</dependency>
<dependency>
<groupId>com.lxs</groupId>
<artifactId>legou-search-instance</artifactId>
<version>${project.version}</version>
</dependency>
<!-- swagger -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<!--elasticsearch-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<!--商品微服务-->
<dependency>
<groupId>com.lxs</groupId>
<artifactId>legou-item-instance</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.lxs</groupId>
<artifactId>legou-common</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
6.2启动器和配置文件
legou-search/legou-search-service/src/main/resources/bootstrap.yml
spring:
application:
name: search-service
# 多个接口上的@FeignClient(“相同服务名”)会报错,overriding is disabled。
# 设置 为true ,即 允许 同名
main:
allow-bean-definition-overriding: true
config-repo/search-service.yml
server:
port: 9006
logging:
#file: demo.log
pattern:
console: "%d - %msg%n"
level:
org.springframework.web: debug
com.lxs: debug
security:
oauth2:
resource:
jwt:
key-uri: http://localhost:9098/oauth/token_key #如果使用JWT,可以获取公钥用于 token 的验签
spring:
data:
elasticsearch:
cluster-name: elasticsearch
cluster-nodes: 192.168.220.110:9300
elasticsearch:
rest:
uris: 192.168.220.110:9200
jackson:
default-property-inclusion: non_null # 配置json处理时忽略空值
legou-search/legou-search-service/src/main/java/com/lxs/legou/search/SearchApplication.java
package com.lxs.legou.search;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.cloud.openfeign.EnableFeignClients;
/**
* @author
* @version 1.0
* @description 搜索启动器
* @createDate 2022/6/8 11:31
**/
@SpringBootApplication //spring boot
@EnableDiscoveryClient //将微服务注册到注册中心
@EnableFeignClients //通过feign调用其他微服务
@EnableCircuitBreaker //开启熔断,微服务容错保护
public class SearchApplication {
public static void main(String[] args) {
SpringApplication.run(SearchApplication.class, args);
}
}
6.3索引库数据格式分析
我们需要商品数据导入索引库,便于用户搜索。我们有SPU和SKU,到底如何保存到索引库?
6.3.1.以结果为导向
大家来看下搜索结果页:
可以看到,每一个搜索结果都有至少1个商品,当我们选择大图下方的小图,商品会跟着变化。
因此,搜索的结果是SPU,即多个SKU的集合。
既然搜索的结果是SPU,那么我们索引库中存储的应该也是SPU,但是却需要包含SKU的信息。
6.3.2.需要什么数据
再来看看页面中有什么数据:
直观能看到的:图片、价格、标题、副标题
暗藏的数据:spu的id,sku的id
另外,页面还有过滤条件:
这些过滤条件也都需要存储到索引库中,包括:
商品分类、品牌、可用来搜索的规格参数等
综上所述,我们需要的数据格式有:
spuId、SkuId、商品分类id、品牌id、图片、价格、商品的创建时间、sku信息集、可搜索的规格参数
6.3.3.最终的数据结构
我们创建一个类,封装要保存到索引库的数据,并设置映射属性:
package com.lxs.legou.search.po;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 搜索数据
* @createDate 2022/6/8 11:42
**/
@Data
@Document(indexName = "goods_legou", type = "docs_legou", shards = 1, replicas = 0)
public class Goods {
@Id
private Long id; // spuId
@Field(type = FieldType.Text, analyzer = "ik_max_word")
private String all; // 所有需要被搜索的信息,包含标题,分类,甚至品牌
@Field(type = FieldType.Keyword, index = false)
private String subTitle;// 卖点 Keyword:不进行分词,主要进行聚合和精确搜索。这里显示副标题
private Long brandId;// 品牌id
private Long cid1;// 1级分类id
private Long cid2;// 2级分类id
private Long cid3;// 3级分类id
private Date createTime;// 创建时间
private List<Long> price;// 价格 价格数组,是所有sku的价格集合。方便根据价格进行筛选过滤
@Field(type = FieldType.Keyword, index = false)
private String skus;// sku信息的json结构 用于页面展示的sku信息,不索引,不搜索。包含skuId、image、price、title字段
private Map<String, Object> specs;// 可搜索的规格参数,key是参数名,值是参数值
}
一些特殊字段解释:
all:用来进行全文检索的字段,里面包含标题、商品分类信息
price:价格数组,是所有sku的价格集合。方便根据价格进行筛选过滤
skus:用于页面展示的sku信息,不索引,不搜索。包含skuId、image、price、title字段
specs:所有规格参数的集合。key是参数名,值是参数值。
例如:我们在specs中存储 内存:4G,6G,颜色为红色,转为json就是:
{
"specs":{
"内存":[4G,6G],
"颜色":"红色"
}
}
当存储到索引库时,elasticsearch会处理为两个字段:
specs.内存:[4G,6G]
specs.颜色:红色
另外, 对于字符串类型,还会额外存储一个字段,这个字段不会分词,用作聚合。
specs.颜色.keyword:红色
ES5.0及以后的版本取消了 string 类型,将原先的 string 类型拆分为 text 和 keyword 两种类型。它们的区别在于 text 会对字段进行分词处理而 keyword 则不会。
当你没有以IndexTemplate等形式为你的索引字段预先指定mapping的话,ES就会使用DynamicMapping,通过推断你传入的文档中字段的值对字段进行动态映射。例如传入的文档中字段price的值为12,那么price将被映射为 long 类型;字段addr的值为"192.168.0.1",那么addr将被映射为 ip 类型。然而对于不满足ip和date格式的普通字符串来说,情况有些不同:ES会将它们映射为text类型,但为了保留对这些字段做精确查询以及聚合的能力,又同时对它们做了keyword类型的映射,作为该字段的fields属性写到_mapping中。例如,当ES遇到一个新的字段"foobar": "somestring"时,会对它做如下的Dynamic Mapping:
{
"foobar": {
"type" "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
又比如商场中的CPU品牌
"specs" : {
"properties" : {
"CPU品牌" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
在之后的查询中使用 specs.CPU品牌 是将 specs.CPU品牌 作为text类型查询,而使用 specs.CPU品牌.keyword 则是将 specs.CPU品牌 作为keyword类型查询。前者会对查询内容做分词处理之后再匹配,而后者则是直接对查询结果做精确匹配。
ES的term query做的是精确匹配而不是分词查询,因此对text类型的字段做term查询将是查不到结果的(除非字段本身经过分词器处理后不变,未被转换或分词)。此时,必须使用 specs.CPU品牌.keyword 来对 specs.CPU品牌 字段以keyword类型进行精确匹配。
6.4 商品微服务提供接口
思路分析:
索引库中的数据来自于数据库,我们不能直接去查询商品的数据库,因为真实开发中,每个微服务都是相互独立的,包括数据库也是一样。所以我们只能调用商品微服务提供的接口服务。
先思考我们需要的数据:
SPU信息
SKU信息
SPU的详情
商品分类名称(拼接all字段)
再思考我们需要哪些服务:
第一:分批查询spu的服务,已经写过。
第二:根据spuId查询sku的服务,已经写过
第三:根据spuId查询SpuDetail的服务,已经写过
第四:根据商品分类id,查询商品分类名称,没写过
第五:根据商品品牌id,查询商品的品牌,没写过
因此我们需要额外提供一个查询商品分类名称的接口。
根据第四点,
6.4.1查询品牌
BrandController
public class BrandController extends BaseController<IBrandService, Brand> {
@ApiOperation(value="根据ids查询", notes = "根据ids查询")
@GetMapping("/list-by-ids")
public List<Brand> selectBrandByIds(@RequestParam("ids") List<Long> ids) {
return service.selectBrandByIds(ids);
}
}
IBrandService
public interface IBrandService extends ICrudService<Brand>{
/**
* 根据商品品牌id,查询商品的品牌
* @param ids
* @return
*/
List<Brand> selectBrandByIds(List<Long> ids);
}
BrandServiceImpl
public class BrandServiceImpl extends CrudServiceImpl<Brand> implements IBrandService {
/**
* 根据商品品牌id,查询商品的品牌
* @param ids
* @return
*/
@Override
public List<Brand> selectBrandByIds(List<Long> ids) {
QueryWrapper<Brand> queryWrapper = Wrappers.<Brand>query().in("id_",ids);
return getBaseMapper().selectList(queryWrapper);
}
}
API:
提供方法根据品牌id查询品牌名称,拼接all
package com.lxs.legou.item.api;
import com.lxs.legou.item.po.Brand;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.List;
@RequestMapping("/brand")
public interface BrandApi {
@ApiOperation(value="根据ids查询品牌", notes = "根据ids查询")
@GetMapping("/list-by-ids")
public List<Brand> selectBrandByIds(@RequestParam("ids") List<Long> ids);
}
legou-search/legou-search-service/src/main/java/com/lxs/legou/search/client/BrandClient.java
package com.lxs.legou.search.client;
import com.lxs.legou.item.api.BrandApi;
import com.lxs.legou.item.po.Brand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RequestMapping;
import java.util.List;
@FeignClient(name = "item-service", fallback = BrandClient.BrandClientFallback.class)
public interface BrandClient extends BrandApi {
@Component
@RequestMapping("/brand-fallback") //这个可以避免容器中requestMapping重复
class BrandClientFallback implements BrandClient {
private static final Logger LOGGER = LoggerFactory.getLogger(BrandClientFallback.class);
@Override
public List<Brand> selectBrandByIds(List<Long> ids) {
LOGGER.info("异常发生,进入fallback方法");
return null;
}
}
}
6.4.2查询分类
CategoryController:
public class CategoryController extends BaseController<ICategoryService,
Category> {
@ApiOperation(value="根据ids查询names", notes = "根据分类id查询名称列表")
@GetMapping("/names")
public ResponseEntity<List<String>> queryNameByIds(@RequestParam("ids") List<Long> ids) {
List<String> names = service.selectNamesByIds(ids);
if (null == names || names.size() == 0) {
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
}
return ResponseEntity.ok(names);
}
}
ICategoryService:
public interface ICategoryService extends ICrudService<Category> {
/**
* 根据分类id查询名称列表
* @param ids
* @return
*/
public List<String> selectNamesByIds(List<Long> ids);
}
CategoryServiceImpl :
public class CategoryServiceImpl extends CrudServiceImpl<Category> implements
ICategoryService {
/**
* 根据分类id查询名称列表
* @param ids
* @return
*/
@Override
public List<String> selectNamesByIds(List<Long> ids) {
QueryWrapper<Category> queryWrapper = Wrappers.
<Category>query().in("id_", ids);
return getBaseMapper().selectList(queryWrapper).stream().map(item ->
item.getTitle()).collect(Collectors.toList());
}
}
CategoryApi:
public interface CategoryApi {
@ApiOperation(value="查询", notes="根据实体条件查询")
@RequestMapping(value = "/list")
public List<Category> list(Category category);
@ApiOperation(value="根据ids查询names", notes = "根据分类id查询名称列表")
@GetMapping("/names")
public List<String> queryNameByIds(@RequestParam("ids") List<Long> ids);
@ApiOperation(value="加载", notes="根据ID加载")
@GetMapping("/edit/{id}")
public Category edit(@PathVariable Long id);
}
CategoryClient:
@FeignClient(name = "item-service", fallback = CategoryClient.CategoryClientFallback.class)
public interface CategoryClient extends CategoryApi {
@Component
@RequestMapping("/category-fallback") //这个可以避免容器中requestMapping重复
class CategoryClientFallback implements CategoryClient {
private static final Logger LOGGER = LoggerFactory.getLogger(CategoryClientFallback.class);
@Override
public List<String> queryNameByIds(List<Long> ids) {
LOGGER.info("异常发生,进入fallback方法");
return null;
}
@Override
public List<Category> list(Category category) {
LOGGER.info("异常发生,进入fallback方法");
return null;
}
@Override
public Category edit(Long id) {
LOGGER.info("异常发生,进入fallback方法");
return null;
}
}
}
6.4.3 查询SKU
legou-search/legou-search-service/src/main/java/com/lxs/legou/search/client/SkuClient.java
@FeignClient(name = "item-service", fallback = SkuClient.SkuClientFallback.class)
public interface SkuClient extends SkuApi {
@Component
@RequestMapping("/sku-fallback")
//这个可以避免容器中requestMapping重复
class SkuClientFallback implements SkuClient {
private static final Logger LOGGER = LoggerFactory.getLogger(SkuClientFallback.class);
@Override
public List<Sku> selectSkusBySpuId(Long spuId) {
LOGGER.error("异常发生,进入fallback方法");
return null;
}
}
}
API根据spuID查询sku列表
@RequestMapping(value = "/sku")
public interface SkuApi {
@ApiOperation(value="查询spu对应的sku", notes="根据spuId查询sku集合")
@GetMapping("/select-skus-by-spuid/{id}")
public List<Sku> selectSkusBySpuId(@PathVariable("id") Long spuId);
}
SkuController:
@RestController
@RequestMapping(value = "/item/sku")
@CrossOrigin// 跨域访问
public class SkuController extends BaseController<ISkuService, Sku> {
@ApiOperation(value="查询spu对应的sku", notes="根据spuId查询sku集合")
@GetMapping("/select-skus-by-spuid/{id}")
public List<Sku> selectSkusBySpuId(@PathVariable("id") Long spuId) {
Sku sku = new Sku();
sku.setSpuId(spuId);
return service.list(sku);
}
}
SkuServiceImpl:
/**
* 重写List方法
* @param entity
* @return
*/
@Override
public List<Sku> list(Sku entity) {
QueryWrapper<Sku> queryWrapper = Wrappers.<Sku>query();
if(entity.getSpuId() != null){
queryWrapper.eq("spu_id_",entity.getSpuId());
}
return super.list(entity);
}
6.4.4 查询SPU
SpuClient
@FeignClient(name = "item-service", fallback = SpuClient.SpuClientFallback.class)
public interface SpuClient extends SpuApi {
@Component
@RequestMapping("/spu-fallback") //这个可以避免容器中requestMapping重复
class SpuClientFallback implements SpuClient {
private static final Logger LOGGER = LoggerFactory.getLogger(SpuClientFallback.class);
@Override
public List<Spu> selectAll() {
LOGGER.error("异常发生,进入fallback方法");
return null;
}
}
}
API查询所有spu
@RequestMapping(value = "/spu")
public interface SpuApi {
@ApiOperation(value="查询所有", notes="查询所有spu")
@GetMapping("/list-all")
public List<Spu> selectAll();
}
SpuController
@ApiOperation(value="查询所有", notes="查询所有spu")
@GetMapping("/list-all")
public List<Spu> selectAll() {
return service.list(new Spu());
}
6.4.5 查询SpuDetail
SpuDetailClient:
@FeignClient(name = "item-service", fallback = SpuDetailClient.SpuDetailFallback.class)
public interface SpuDetailClient extends SpuDetailApi {
@Component
@RequestMapping("/spu-detail-fallback") //这个可以避免容器中requestMapping重复
class SpuDetailFallback implements SpuDetailClient {
private static final Logger LOGGER = LoggerFactory.getLogger(SpuDetailFallback.class);
@Override
public SpuDetail edit(Long id) {
System.out.println("异常发生,进入fallback方法");
return null;
}
}
}
api根据id 查询SpuDetail
@RequestMapping(value = "/item/spu-detail")
public interface SpuDetailApi {
/**
* 加载
*
* @param id
* @return
* @throws Exception
*/
@ApiOperation(value="加载", notes="根据ID加载")
@GetMapping("/edit/{id}")
public SpuDetail edit(@PathVariable Long id);
}
6.4.6 查询规格参数
SpecParamClient
@FeignClient(name = "item-service", fallback = SpecParamClient.SpecParamClientFallback.class)
public interface SpecParamClient extends SpecParamApi {
@Component
@RequestMapping("/param-fallback") //这个可以避免容器中requestMapping重复
class SpecParamClientFallback implements SpecParamClient {
private static final Logger LOGGER = LoggerFactory.getLogger(SpecParamClientFallback.class);
@Override
public List<SpecParam> selectSpecParamApi(SpecParam entity) {
LOGGER.error("异常发生,进入fallback方法");
return null;
}
}
}
api根据实体条件查询规格参数
@RequestMapping(value = "/item/param")
public interface SpecParamApi {
@ApiOperation(value="查询", notes="根据实体条件查询参数")
@PostMapping(value = "/select-param-by-entity", consumes = "application/json")
public List<SpecParam> selectSpecParamApi(@RequestBody SpecParam entity);
}
SpecParamController
public class SpecParamController extends BaseController<ISpecParamService,
SpecParam> {
@ApiOperation(value="查询", notes="根据实体条件查询参数")
@PostMapping(value = "/select-param-by-entity")
public List<SpecParam> selectSpecParamApi(@RequestBody SpecParam entity) {
return service.list(entity);
}
}
6.5 导入数据
导入数据只做一次,以后的更新删除等操作通过消息队列或者canal来操作索引库
6.5.1创建GoodsRepository
package com.lxs.legou.search.dao;
import com.lxs.legou.search.po.Goods;
importorg.springframework.data.elasticsearch.repository.ElasticsearchRepository;
public interface GoodsDao extends ElasticsearchRepository<Goods, Long> {
}
6.5.2.创建索引
我们新建一个测试类,在里面进行数据的操作:
import static org.junit.Assert.*;
import com.lxs.legou.search.SearchApplication;
import com.lxs.legou.search.po.Goods;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.test.context.junit4.SpringRunner;
/**
* 索引测试
*/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = SearchApplication.class)
public class ElasticSearchTest {
@Autowired
private ElasticsearchTemplate elasticsearchTemplate;
@Test
public void createIndex() {
//创建索引
this.elasticsearchTemplate.createIndex(Goods.class);
//配置映射
this.elasticsearchTemplate.putMapping(Goods.class);
}
}
通过kibana查看:
6.5.3.导入数据
导入数据其实就是查询数据,然后把查询到的Spu转变为Goods来保存,因此我们先编写一个IndexService,然后在里面定义一个方法, 把Spu转为Goods
package com.lxs.legou.search.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.lxs.legou.common.utils.JsonUtils;
import com.lxs.legou.item.po.Sku;
import com.lxs.legou.item.po.SpecParam;
import com.lxs.legou.item.po.Spu;
import com.lxs.legou.item.po.SpuDetail;
import com.lxs.legou.search.client.CategoryClient;
import com.lxs.legou.search.client.SkuClient;
import com.lxs.legou.search.client.SpecParamClient;
import com.lxs.legou.search.client.SpuDetailClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
/**
* @author
* @version 1.0
* @description 管理索引的服务类
* 把spu转换成goods,删除索引库
* @createDate 2022/6/9 8:11
**/
@Service
public class IndexService {
@Autowired
private GoodsDao goodsDao;
@Autowired
private CategoryClient categoryClient;
@Autowired
private SpecParamClient specParamClient;
@Autowired
private SkuClient skuClient;
@Autowired
private SpuDetailClient spuDetailClient;
/**
* 根据spu构建索引类型
*
* @param spu
* @return
*/
public Goods buildGoods(Spu spu) {
Long id = spu.getId();
//准备数据
//商品分类名称 拼接商品分类名称得到all数据
List<String> names =
this.categoryClient.queryNameByIds(Arrays.asList(spu.getCid1(), spu.getCid2(),
spu.getCid3()));
String all = spu.getTitle() + " " + StringUtils.join(names, " ");
//sku集合
List<Sku> skus = skuClient.selectSkusBySpuId(spu.getId()); // 查询所有sku
//处理sku
//把商品价格取出单独存放,便于展示
List<Long> prices = new ArrayList<>();
List<Map<String, Object>> skuList = new ArrayList<>();
for (Sku sku : skus) {
prices.add(sku.getPrice());
Map<String, Object> skuMap = new HashMap<>();
skuMap.put("id", sku.getId());
skuMap.put("title", sku.getTitle());
skuMap.put("image", StringUtils.isBlank(sku.getImages()) ? "" :
sku.getImages().split(",")[0]);
skuMap.put("price", sku.getPrice());
skuList.add(skuMap);
}
//spec
Map<String, Object> specs = new HashMap<>();
//spuDetail
SpuDetail spuDetail = spuDetailClient.edit(spu.getId());
//通用规格参数值
Map<String, String> genericMap =
JsonUtils.parseMap(spuDetail.getGenericSpec(), String.class, String.class);
//特有规格参数的值
Map<String, List<String>> specialMap =
JsonUtils.nativeRead(spuDetail.getSpecialSpec(), new TypeReference<Map<String,
List<String>>>() {
});
//查询分类对应的规格参数
SpecParam specParam = new SpecParam();
specParam.setCid(spu.getCid3());
specParam.setSearching(true);
List<SpecParam> params = specParamClient.selectSpecParamApi(specParam);
for (SpecParam param : params) {
//今后显示的名称
String name = param.getName();//品牌,机身颜色
//通用参数
Object value = null;
if (param.getGeneric()) {
//通用参数
value = genericMap.get(name);
if (param.getNumeric()) {
//数值类型需要加分段
value = this.chooseSegment(value.toString(), param);
}
} else {//特有参数
value = specialMap.get(name);
}
if (null == value) {
value = "其他";
}
specs.put(name, value);
}
// 把相关数据存入goods
Goods goods = new Goods();
goods.setId(spu.getId());
//这里如果要加品牌,可以再写个BrandClient,根据id查品牌
goods.setAll(all);
goods.setSubTitle(spu.getSubTitle());
goods.setBrandId(spu.getBrandId());
goods.setCid1(spu.getCid1());
goods.setCid2(spu.getCid2());
goods.setCid3(spu.getCid3());
goods.setCreateTime(spu.getCreateTime());
goods.setPrice(prices);
goods.setSkus(JsonUtils.serialize(skuList));
goods.setSpecs(specs);
return goods;
}
/**
* 数值类型加分段
* @param value
* @param p
* @return
*/
private String chooseSegment(String value, SpecParam p) {
double val = NumberUtils.toDouble(value);
String result = "其它";
// 保存数值段
for (String segment : p.getSegments().split(",")) {
String[] segs = segment.split("-");
// 获取数值范围
double begin = NumberUtils.toDouble(segs[0]);
double end = Double.MAX_VALUE;
if (segs.length == 2) {
end = NumberUtils.toDouble(segs[1]);
}
// 判断是否在范围内
if (val >= begin && val < end) {
if (segs.length == 1) {
result = segs[0] + p.getUnit() + "以上";
} else if (begin == 0) {
result = segs[1] + p.getUnit() + "以下";
} else {
result = segment + p.getUnit();//4.5 4-5英寸
}
break;
}
}
return result;
}
/**
* 根据商品id删除索引
* @param id
*/
public void deleteIndex(Long id) {
goodsDao.deleteById(id);
}
}
因为过滤参数中有一类比较特殊,就是数值区间:
所以我们在存入时要进行处理:
/**
* 数值类型加分段
* @param value
* @param p
* @return
*/
private String chooseSegment(String value, SpecParam p) {
double val = NumberUtils.toDouble(value);
String result = "其它";
// 保存数值段
for (String segment : p.getSegments().split(",")) {
String[] segs = segment.split("-");
// 获取数值范围
double begin = NumberUtils.toDouble(segs[0]);
double end = Double.MAX_VALUE;
if (segs.length == 2) {
end = NumberUtils.toDouble(segs[1]);
}
// 判断是否在范围内
if (val >= begin && val < end) {
if (segs.length == 1) {
result = segs[0] + p.getUnit() + "以上";
} else if (begin == 0) {
result = segs[1] + p.getUnit() + "以下";
} else {
result = segment + p.getUnit();//4.5 4-5英寸
}
break;
}
}
return result;
}
然后编写一个测试类,循环查询Spu,然后调用IndexService中的方法,把SPU变为Goods,然后写入索引库:
import com.lxs.legou.item.po.Spu;
import com.lxs.legou.search.SearchApplication;
import com.lxs.legou.search.client.SpuClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import com.lxs.legou.search.service.IndexService;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author
* @version 1.0
* @description 测试搜索
* @createDate 2022/6/9 9:17
**/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = SearchApplication.class)
public class ESLoadDataTest {
@Autowired
private IndexService indexService;
@Autowired
private SpuClient spuClient;
@Autowired
private GoodsDao goodsDao;
@Test
public void loadData() {
// 查询spu
// PageResult<SpuBO> result = this.goodsClient.querySpuByPage(page,rows, true, null);
// List<SpuBO> spus = result.getItems();
List<Spu> spus = spuClient.selectAll();
// spu转为goods
List<Goods> goods = spus.stream().map(spu ->
this.indexService.buildGoods(spu)).collect(Collectors.toList());
// 把goods放入索引库
goodsDao.saveAll(goods);
}
}
通过kibana查询, 可以看到数据成功导入:
六,商品搜索
6.1 前端实现
6.1.1 发送请求
前端搜索子组件Search调用searchGoodList方法
<Search @onSearch="searchGoodList"></Search>
searchGoodList方法,根据用户输入修改搜索对象中的key
//搜索输入框搜索
searchGoodList(data) {
this.search.key = data
}
search对象的结构
search: {
key: "", // 搜索页面的关键字
page:1,
sortBy:"", //根据谁排序
descending:false, //升序还是降序
filter:{} //规律条件
}
search属性的侦听器
watch:{
search:{
deep:true,
handler(val,old){
if(!old || !old.key){
// 如果旧的search值为空,或者search中的key为空,证明是第一次
return;
}
this.searchBy();
}
}
}
发送搜索请求的searchBy方法
//搜索
searchBy() {
instance.post(`/search/query`, this.search, {
headers: {
'Content-Type': 'application/json;charset=UTF-8'
}
}).then(response => {
//初始化skus属性,并且让商品的默认选择选第0个
response.data.items.forEach(goods => {
//把之际取到的字符串转换成json
goods.skus = JSON.parse(goods.skus).sort();
//表示选中的sku。默认选中第0个
goods.selected = goods.skus[0];
});
//从响应数据中获取总的条目数以及总页数
this.total = response.data.total;
this.totalPage = response.data.totalPage;
this.filters = [];
this.filters.push({
k:"cid3",
options:response.data.categories
});
this.filters.push({
k:"brandId",
options:response.data.brands
});
response.data.specs.forEach(spec=>{
spec.options = spec.options.map(option=>({name:option}));
this.filters.push(spec);
});
//当前页面上的所有的spu
this.goodsList = response.data.items;
}).catch(error => {
console.log(error)
})
}
发送给搜索微服务的请求数据
6.1.2 处理结果
搜索微服务返回结果分析
vue devtools中监控到的数据结构
搜索结果展示
<div class="goods-list">
<!--<div class="goods-show-info" v-for="(item, index) in orderGoodsList"
:key="index">-->
<div class="goods-show-info" v-for="(item, index) in goodsList"
:key="index">
<div class="goods-show-img">
<router-link to="/goodsDetail"><img :src="item.selected.image"
height="200"/></router-link>
<ul class="skus">
<li :class="{selected : sku.id === item.selected.id}" v-for="(sku,i)
in item.skus" :key="i"
@click="item.selected = sku">
<img :src="sku.image">
</li>
</ul>
</div>
<div class="goods-show-price">
<span>
<Icon type="social-yen text-danger"></Icon>
<span class="seckill-price text-danger">{{item.selected.price}}</span>
</span>
</div>
<div class="goods-show-detail">
<span>{{item.selected.title}}</span>
</div>
<div class="goods-show-num">
已有<span>10</span>人评价
</div>
<div class="goods-show-seller">
legou-search/legou-search-instance/src/main/java/com/lxs/legou/search/po/SearchRequest.java
<span>自营</span>
</div>
</div>
</div>
</div>
6.2 后端实现
6.2.1 实体类
搜索请求对象
SearchRequest对象对应前端的search搜索对象
legousearch/legousearchinstance/src/main/java/com/lxs/legou/search/po/SearchRequest.java
package com.lxs.legou.search.po;
import java.util.HashMap;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 搜索请求对象
* SearchRequest对象对应前端的search搜索对象
* @createDate 2022/6/10 14:55
**/
public class SearchRequest {
private String key;// 搜索条件
private Integer page;// 当前页
private String sortBy;//根据谁排序
private Boolean descending; //升序还是降序
private Map<String,String> filter = new HashMap<>();
private static final Integer DEFAULT_SIZE = 20;// 每页大小,不从页面接收,而是固定大小
private static final Integer DEFAULT_PAGE = 1;// 默认页
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public Integer getPage() {
if(page == null){
return DEFAULT_PAGE;
}
// 获取页码时做一些校验,不能小于1
return Math.max(DEFAULT_PAGE, page);
}
public void setPage(Integer page) {
this.page = page;
}
public Integer getSize() {
return DEFAULT_SIZE;
}
public String getSortBy() {
return sortBy;
}
public void setSortBy(String sortBy) {
this.sortBy = sortBy;
}
public Boolean getDescending() {
return descending;
}
public void setDescending(Boolean descending) {
this.descending = descending;
}
public Map<String, String> getFilter() {
return filter;
}
public void setFilter(Map<String, String> filter) {
this.filter = filter;
}
}
搜索结果实体类
SearchResult对象对应前端的goodsList和filter对象,是这个对象的数据来源
legou-search/legou-search-instance/src/main/java/com/lxs/legou/search/po/SearchResult.java.
package com.lxs.legou.search.po;
import com.lxs.legou.item.po.Brand;
import com.lxs.legou.item.po.Category;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 搜索结果实体类
* SearchResult对象对应前端的goodsList和filter对象,是这个对象的数据来源
* @createDate 2022/6/10 15:22
**/
@Data
public class SearchResult {
private Long total; //总行数
private Long totalPage; //总页数
private List items; //当前页数据
private List<Category> categories;
private List<Brand> brands;
private List<Map<String, Object>> specs;
public SearchResult() {
}
public SearchResult(Long total, Long totalPage, List items, List<Category>
categories, List<Brand> brands, List<Map<String, Object>> specs) {
this.total = total;
this.totalPage = totalPage;
this.items = items;
this.categories = categories;
this.brands = brands;
this.specs = specs;
}
public Long getTotal() {
return total;
}
public void setTotal(Long total) {
this.total = total;
}
public Long getTotalPage() {
return totalPage;
}
public void setTotalPage(Long totalPage) {
this.totalPage = totalPage;
}
public List getItems() {
return items;
}
public void setItems(List items) {
this.items = items;
}
public List<Category> getCategories() {
return categories;
}
public void setCategories(List<Category> categories) {
this.categories = categories;
}
public List<Brand> getBrands() {
return brands;
}
public void setBrands(List<Brand> brands) {
this.brands = brands;
}
public List<Map<String, Object>> getSpecs() {
return specs;
}
public void setSpecs(List<Map<String, Object>> specs) {
this.specs = specs;
}
}
6.2.2 业务类
创建业务类,对于用户输入的key进行基本的搜索
legou-search/legou-search-service/src/main/java/com/lxs/legou/search/service/SearchService.java
package com.lxs.legou.search.service;
import com.lxs.legou.search.client.BrandClient;
import com.lxs.legou.search.client.CategoryClient;
import com.lxs.legou.search.client.SpecParamClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import com.lxs.legou.search.po.SearchRequest;
import com.lxs.legou.search.po.SearchResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 商城搜索服务层
* @createDate 2022/6/10 15:33
**/
@Service
public class SearchService {
@Autowired
private GoodsDao goodsRepository;
/* @Autowired
private BrandClient brandClient;
@Autowired
private CategoryClient categoryClient;
@Autowired
private SpecParamClient specificationClient;*/
//private Logger logger = LoggerFactory.getLogger(SearchService.class);
public SearchResult search(SearchRequest searchRequest) {
String key = searchRequest.getKey();
if (key == null) {
return null;
}
//查询构建工具
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//添加了查询的过滤,只要这些字段
queryBuilder.withSourceFilter(new FetchSourceFilter(new String[]
{"id", "subTitle", "skus"}, null));
//获取基本的查询条件
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("all", searchRequest.getKey()));
//QueryBuilder basicQuery = buildBasicQueryWithFilter(searchRequest);
//把查询条件添加到构建器中(这里仅仅是我的查询条件)
queryBuilder.withQuery(boolQueryBuilder);
Page<Goods> goodsResult = goodsRepository.search(queryBuilder.build());
//计算总页数
long total = goodsResult.getTotalElements();
long totalPages = goodsResult.getTotalPages(); //(total + size - 1) / size;
return new SearchResult(total, totalPages, goodsResult.getContent(), null, null, null);
}
/* //这个方法用来构建查询条件以及过滤条件
private QueryBuilder buildBasicQueryWithFilter(SearchRequest searchRequest) {
//构造布尔查询
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.matchQuery("all", searchRequest.getKey()));
}*/
}
6.2.3 控制器
package com.lxs.legou.search.controller;
import com.lxs.legou.search.po.SearchRequest;
import com.lxs.legou.search.po.SearchResult;
import com.lxs.legou.search.service.SearchService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
/**
* @author
* @version 1.0
* @description 搜索控制层
* @createDate 2022/6/10 16:08
**/
@RestController
@CrossOrigin// 跨域访问
public class SearchController {
@Autowired
private SearchService searchService;
@PostMapping("/query")
public ResponseEntity<SearchResult> queryGoodsByPage(@RequestBody SearchRequest searchRequest) {
SearchResult result = searchService.search(searchRequest);
if (result == null) {
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
}
return ResponseEntity.ok(result);
}
}
6.2.4 测试
注意配置请求头Content-Type=application/json;charset=UTF-8
6.3 品牌统计
用户搜索的时候,除了使用分类搜索外,还有可能使用品牌搜索,所以我们还需要显示品牌数据和规格数据,品牌数据和规格数据的显示比较容易,都可以考虑使用分类统计的方式进行分组实现。
6.3.1 品牌统计分析
看下面的SQL语句,我们在执行搜索的时候,第1条SQL语句是执行搜,第2条语句是根据品牌名字分组查看有多少品牌,大概执行了2个步骤就可以获取数据结果以及品牌统计,我们可以发现他们的搜索条件完全一样。
-- 查询所有
SELECT * FROM spu_ WHERE name LIKE '%手机%';
-- 根据品牌名字分组查询
SELECT brand_id_ FROM spu_ WHERE name LIKE '%手机%' GROUP BY brand_id_;
我们每次执行搜索的时候,需要显示商品品牌名称,这里要显示的品牌名称其实就是符合搜素条件的所有商品的品牌集合,我们可以按照上面的实现思路,使用ES根据分组名称做一次分组查询即可实现。
6.3.2 品牌分组统计实现
修改search微服务的legou-search/legou-search-service/src/main/java/com/lxs/legou/search/service/SearchService.java类,添加一个品牌分组搜索
queryBuilder.addAggregation(AggregationBuilders.terms(brandAggsName).field("brandId"));
整体代码如下:
package com.lxs.legou.search.service;
import com.lxs.legou.item.po.Brand;
import com.lxs.legou.search.client.BrandClient;
import com.lxs.legou.search.client.CategoryClient;
import com.lxs.legou.search.client.SpecParamClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import com.lxs.legou.search.po.SearchRequest;
import com.lxs.legou.search.po.SearchResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 商城搜索服务层
* @createDate 2022/6/10 15:33
**/
@Service
public class SearchService {
@Autowired
private GoodsDao goodsRepository;
@Autowired
private BrandClient brandClient;
/*@Autowired
private CategoryClient categoryClient;
@Autowired
private SpecParamClient specificationClient;*/
//private Logger logger = LoggerFactory.getLogger(SearchService.class);
public SearchResult search(SearchRequest searchRequest) {
String key = searchRequest.getKey();
if (key == null) {
return null;
}
//查询构建工具
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//添加了查询的过滤,只要这些字段
queryBuilder.withSourceFilter(new FetchSourceFilter(new String[]
{"id", "subTitle", "skus"}, null));
//获取基本的查询条件
QueryBuilder basicQuery = buildBasicQueryWithFilter(searchRequest);
//把查询条件添加到构建器中(这里仅仅是我的查询条件)
queryBuilder.withQuery(basicQuery);
// 品牌聚合查询
String brandAggsName ="brands";
queryBuilder.addAggregation(AggregationBuilders.terms(brandAggsName).field("brandId"));
// 查询结果
AggregatedPage<Goods> goodsResult = (AggregatedPage<Goods>) goodsRepository.search(queryBuilder.build());
// 品牌聚合结果
List<Brand> brands = getBrandAgg(brandAggsName,goodsResult);
//计算总页数
long total = goodsResult.getTotalElements();
long totalPages = goodsResult.getTotalPages(); //(total + size - 1) / size;
return new SearchResult(total, totalPages, goodsResult.getContent(), null, brands, null);
}
//这个方法用来构建查询条件以及过滤条件
private QueryBuilder buildBasicQueryWithFilter(SearchRequest searchRequest) {
//构造布尔查询
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.matchQuery("all", searchRequest.getKey()));
return queryBuilder;
}
private List<Brand> getBrandAgg(String aggName, AggregatedPage<Goods>
result){
try {
LongTerms longTerms1 = (LongTerms) result.getAggregation(aggName);
List<Long> brandIds = new ArrayList<>();
for (LongTerms.Bucket bucket : longTerms1.getBuckets()) {
brandIds.add(bucket.getKeyAsNumber().longValue());
}
// return this.brandClient.queryBrandByIds(brandIds);
return this.brandClient.selectBrandByIds(brandIds);
} catch (Exception e) {
//logger.error("解析品牌数据错误:{}",e);
e.printStackTrace();
}
return null;
}
}
使用kibana查询的DSL语句
GET /goods_legou/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"all": "手机"
}
}
]
}
},
"aggs": {
"brands": {
"terms": {
"field": "brandId"
}
}
}
}
6.3.3 测试
使用PostMan请求http://localhost:9006/query
6.4 分类统计
用户搜索的时候,除了使用分类搜索外,还有可能使用品牌搜索,所以我们还需要显示品牌数据和规格数据,品牌数据和规格数据的显示比较容易,都可以考虑使用分类统计的方式进行分组实现。
6.4.1 分类统计分析
看下面的SQL语句,我们在执行搜索的时候,第1条SQL语句是执行搜,第2条语句是根据品牌名字分组查看有多少品牌,大概执行了2个步骤就可以获取数据结果以及品牌统计,我们可以发现他们的搜索条件完全一样。
-- 查询所有
SELECT * FROM spu_ WHERE name LIKE '%手机%';
-- 根据分类名字分组查询
SELECT cid3 FROM spu_ WHERE name LIKE '%手机%' GROUP BY cid3;
我们每次执行搜索的时候,需要显示商品品牌名称,这里要显示的品牌名称其实就是符合搜素条件的所有商品的品牌集合,我们可以按照上面的实现思路,使用ES根据分组名称做一次分组查询即可实现。
6.4.2分类分组统计实现
修改search微服务的legou-search/legou-search-service/src/main/java/com/lxs/legou/search/service/SearchService.java类,添加一个品牌分组搜索
queryBuilder.addAggregation(AggregationBuilders.terms(categoryAggsName).field("cid3"));
整体代码如下:
package com.lxs.legou.search.service;
import com.lxs.legou.item.po.Brand;
import com.lxs.legou.item.po.Category;
import com.lxs.legou.search.client.BrandClient;
import com.lxs.legou.search.client.CategoryClient;
import com.lxs.legou.search.client.SpecParamClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import com.lxs.legou.search.po.SearchRequest;
import com.lxs.legou.search.po.SearchResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 商城搜索服务层
* @createDate 2022/6/10 15:33
**/
@Service
public class SearchService {
@Autowired
private GoodsDao goodsRepository;
@Autowired
private BrandClient brandClient;
@Autowired
private CategoryClient categoryClient;
/* @Autowired
private SpecParamClient specificationClient;*/
//private Logger logger = LoggerFactory.getLogger(SearchService.class);
public SearchResult search(SearchRequest searchRequest) {
String key = searchRequest.getKey();
if (key == null) {
return null;
}
//查询构建工具
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//添加了查询的过滤,只要这些字段
queryBuilder.withSourceFilter(new FetchSourceFilter(new String[]
{"id", "subTitle", "skus"}, null));
//获取基本的查询条件
QueryBuilder basicQuery = buildBasicQueryWithFilter(searchRequest);
//把查询条件添加到构建器中(这里仅仅是我的查询条件)
queryBuilder.withQuery(basicQuery);
// 品牌聚合查询
String brandAggsName ="brands";
String categoryAggsName ="categories";
queryBuilder.addAggregation(AggregationBuilders.terms(brandAggsName).field("brandId"));
queryBuilder.addAggregation(AggregationBuilders.terms(categoryAggsName).field("cid3"));
// 查询结果
AggregatedPage<Goods> goodsResult = (AggregatedPage<Goods>) goodsRepository.search(queryBuilder.build());
// 聚合结果
List<Brand> brands = getBrandAgg(brandAggsName,goodsResult);
List<Category> categories = getCategoryAgg(categoryAggsName,goodsResult);
//计算总页数
long total = goodsResult.getTotalElements();
long totalPages = goodsResult.getTotalPages(); //(total + size - 1) / size;
return new SearchResult(total, totalPages, goodsResult.getContent(), categories, brands, null);
}
//这个方法用来构建查询条件以及过滤条件
private QueryBuilder buildBasicQueryWithFilter(SearchRequest searchRequest) {
//构造布尔查询
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.matchQuery("all", searchRequest.getKey()));
return queryBuilder;
}
private List<Brand> getBrandAgg(String aggName, AggregatedPage<Goods>
result){
try {
LongTerms longTerms1 = (LongTerms) result.getAggregation(aggName);
List<Long> brandIds = new ArrayList<>();
for (LongTerms.Bucket bucket : longTerms1.getBuckets()) {
brandIds.add(bucket.getKeyAsNumber().longValue());
}
// return this.brandClient.queryBrandByIds(brandIds);
return this.brandClient.selectBrandByIds(brandIds);
} catch (Exception e) {
//logger.error("解析品牌数据错误:{}",e);
e.printStackTrace();
}
return null;
}
private List<Category> getCategoryAgg(String categoryAggsName, AggregatedPage<Goods> goodsResult) {
LongTerms longTerms = (LongTerms)
goodsResult.getAggregation(categoryAggsName);
List<Long> categoryIds = new ArrayList<>();
for (LongTerms.Bucket bucket : longTerms.getBuckets()) {
categoryIds.add(bucket.getKeyAsNumber().longValue());
}
List<String> names = this.categoryClient.queryNameByIds(categoryIds);
List<Category> categories = new ArrayList<>();
for (int i = 0; i < names.size(); i++) {
Category category =new Category();
category.setId(categoryIds.get(i));
// category.setName(names.get(i));
category.setTitle(names.get(i));
categories.add(category);
}
return categories;
}
}
使用kibana执行DSL语句
GET /goods_legou/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"all": "手机"
}
}
]
}
},
"aggs": {
"brands": {
"terms": {
"field": "brandId"
}
},
"categorys": {
"terms": {
"field": "cid3"
}
}
}
}
6.4.3 测试
使用PostMan请求http://localhost:9006/query
6.5规格统计
用户搜索的时候,除了使用分类、品牌搜索外,还有可能使用规格搜索,所以我们还需要显示规格数据,规格数据的显示相比上面2种实现略微较难一些,需要对数据进行处理,我们也可以考虑使用分类统计和品牌统计的方式进行分组实现。
6.5.1 规格统计分析
看下面的SQL语句,我们在执行搜索的时候,第1条SQL语句是执行搜,第2条语句是根据规格分组查看有多少规格,大概执行了2个步骤就可以获取数据结果以及规格统计,我们可以发现他们的搜索条件完全一样。
-- 查询所有
SELECT * FROM spu_detail_ WHERE name LIKE '%手机%';
-- 根据规格名字分组查询
SELECT spec FROM spu_detail WHERE name LIKE '%手机%' GROUP BY spec;
6.5.2 规格统计分组实现
修改search微服务的legou-search/legou-searchservice/src/main/java/com/lxs/legou/search/service/SearchService.java类,添加一个规格分组搜索.
List<Map<String,Object>> specs = null;
/*
- 当分类聚合结果为1个统计规格参数
- 根据分类查询当前分类的搜索的规格参数
- 创建NativeQueryBuilder,使用上面搜索一样的条件
- 循环上面可搜索规格参数,依次添加聚合
- 处理结果k:参数名,options:聚合的结果数组
*/
if (categories.size()==1){
specs = getSpecs(categories.get(0).getId(),basicQuery);
}
整体代码如下:
//规格参数的聚合应该和查询关联
private List<Map<String, Object>> getSpecs(Long cid, QueryBuilder query) {
List<Map<String,Object>> specs = new ArrayList<>();
// List<SpecParam> specParams =this.specificationClient.querySpecParam(null, cid, true, null);
SpecParam sp = new SpecParam();
sp.setCid(cid);
sp.setSearching(true);
List<SpecParam> specParams =
this.specificationClient.selectSpecParamApi(sp);
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//在做聚合之前先做查询,只有符合条件的规格参数才应该被查出来
queryBuilder.withQuery(query);
for (SpecParam specParam : specParams) {
String name = specParam.getName();//操作系统,cpu核数
queryBuilder.addAggregation(AggregationBuilders.terms(name).field("specs."+name +".keyword"));
}
AggregatedPage<Goods> aggs = (AggregatedPage<Goods>)
this.goodsRepository.search(queryBuilder.build());
Map<String, Aggregation> stringAggregationMap =
aggs.getAggregations().asMap();
for (SpecParam specParam : specParams) {
Map<String,Object> spec = new HashMap<>();
String name = specParam.getName();
if (stringAggregationMap.get(name) instanceof StringTerms) {
StringTerms stringTerms = (StringTerms) stringAggregationMap.get(name);
List<String> val = new ArrayList<>();
for (StringTerms.Bucket bucket : stringTerms.getBuckets()) {
val.add(bucket.getKeyAsString());
}
spec.put("k",name);//内存,存储空间,屏幕尺寸
spec.put("options",val);
specs.add(spec);
}
}
return specs;
}
对应kibana中的DSL查询
GET /goods_legou2/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"all": "手机"
}
}
]
}
},
"aggs": {
"brands": {
"terms": {
"field": "brandId"
}
},
"categorys": {
"terms": {
"field": "cid3"
}
},
"CPU品牌": {
"terms": {
"field": "specs.CPU品牌.keyword"
}
},
"CPU核数": {
"terms": {
"field": "specs.CPU核数.keyword"
}
}
}
}
6.6条件过滤
//这个方法用来构建查询条件以及过滤条件
private QueryBuilder buildBasicQueryWithFilter(SearchRequest searchRequest) {
//构造布尔查询
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.matchQuery("all",searchRequest.getKey()));
//给这个查询加过滤
// 过滤条件构建器
BoolQueryBuilder filterQueryBuilder = QueryBuilders.boolQuery();
//取出map中的实体
for (Map.Entry<String, String> entry : searchRequest.getFilter().entrySet())
{
String key = entry.getKey();
String value = entry.getValue();
// 商品分类和品牌不用前后加修饰
if (key != "cid3" && key != "brandId") {
key = "specs." + key + ".keyword";
}
// 字符串类型,进行term查询
filterQueryBuilder.must(QueryBuilders.termQuery(key, value));
}
return queryBuilder.filter(filterQueryBuilder);
}
以上代码加.keyword的解释如下:
ES5.0及以后的版本取消了 string 类型,将原先的 string 类型拆分为 text 和 keyword 两种类型。它们的区别在于 text 会对字段进行分词处理而 keyword 则不会。
当你没有以IndexTemplate等形式为你的索引字段预先指定mapping的话,ES就会使用DynamicMapping,通过推断你传入的文档中字段的值对字段进行动态映射。例如传入的文档中字段price的值为12,那么price将被映射为 long 类型;字段addr的值为"192.168.0.1",那么addr将被映射为 ip 类型。然而对于不满足ip和date格式的普通字符串来说,情况有些不同:ES会将它们映射为text类型,但为了保留对这些字段做精确查询以及聚合的能力,又同时对它们做了keyword类型的映射,作为该字段的fields属性写到_mapping中。例如,当ES遇到一个新的字段"foobar": "somestring"时,会对它做如下的Dynamic Mapping:
{
"foobar": {
"type" "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
又比如商场中的CPU品牌
"specs" : {
"properties" : {
"CPU品牌" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
在之后的查询中使用 specs.CPU品牌 是将 specs.CPU品牌 作为text类型查询,而使用 specs.CPU品牌.keyword 则是将 specs.CPU品牌 作为keyword类型查询。前者会对查询内容做分词处理之后再匹配,而后者则是直接对查询结果做精确匹配。
ES的term query做的是精确匹配而不是分词查询,因此对text类型的字段做term查询将是查不到结果的(除非字段本身经过分词器处理后不变,未被转换或分词)。此时,必须使用 specs.CPU品牌.keyword 来对 specs.CPU品牌 字段以keyword类型进行精确匹配。
使用PostMan测试:
6.7分页实现
后台代码
Integer page = searchRequest.getPage() - 1;// page 从0开始
Integer size = searchRequest.getSize();
//把分页条件条件到构建器中
queryBuilder.withPageable(PageRequest.of(page,size));
前台代码
<Page :total="total" :page-size="20" @on-change="changePage"></Page>
changePage(index) {
this.search.page = index;
}
6.8 排序
后台代码
//获取排序的条件
String sortBy = searchRequest.getSortBy();
Boolean desc = searchRequest.getDescending();
if (StringUtils.isNotBlank(sortBy)){
//把排序条件加给构建器
queryBuilder.withSort(SortBuilders.fieldSort(sortBy).order(desc ?
SortOrder.DESC : SortOrder.ASC));
}
完整代码:
package com.lxs.legou.search.service;
import com.lxs.legou.item.po.Brand;
import com.lxs.legou.item.po.Category;
import com.lxs.legou.item.po.SpecParam;
import com.lxs.legou.search.client.BrandClient;
import com.lxs.legou.search.client.CategoryClient;
import com.lxs.legou.search.client.SpecParamClient;
import com.lxs.legou.search.dao.GoodsDao;
import com.lxs.legou.search.po.Goods;
import com.lxs.legou.search.po.SearchRequest;
import com.lxs.legou.search.po.SearchResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @author
* @version 1.0
* @description 商城搜索服务层
* @createDate 2022/6/10 15:33
**/
@Service
public class SearchService {
@Autowired
private GoodsDao goodsRepository;
@Autowired
private BrandClient brandClient;
@Autowired
private CategoryClient categoryClient;
@Autowired
private SpecParamClient specificationClient;/**/
private Logger logger = LoggerFactory.getLogger(SearchService.class);
public SearchResult search(SearchRequest searchRequest) {
String key = searchRequest.getKey();
if (key == null) {
return null;
}
//查询构建工具
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//添加了查询的过滤,只要这些字段
queryBuilder.withSourceFilter(new FetchSourceFilter(new String[]
{"id", "subTitle", "skus"}, null));
//获取基本的查询条件
QueryBuilder basicQuery = buildBasicQueryWithFilter(searchRequest);
//把查询条件添加到构建器中(这里仅仅是我的查询条件)
queryBuilder.withQuery(basicQuery);
Integer page = searchRequest.getPage() - 1;// page 从0开始
Integer size = searchRequest.getSize();
//把分页条件条件到构建器中
queryBuilder.withPageable(PageRequest.of(page,size));
//获取排序的条件
String sortBy = searchRequest.getSortBy();
Boolean desc = searchRequest.getDescending();
if (StringUtils.isNotBlank(sortBy)){
//把排序条件加给构建器
queryBuilder.withSort(SortBuilders.fieldSort(sortBy).order(desc ?
SortOrder.DESC : SortOrder.ASC));
}
// 品牌聚合查询
String brandAggsName ="brands";
String categoryAggsName ="categories";
queryBuilder.addAggregation(AggregationBuilders.terms(brandAggsName).field("brandId"));
queryBuilder.addAggregation(AggregationBuilders.terms(categoryAggsName).field("cid3"));
// 查询结果
AggregatedPage<Goods> goodsResult = (AggregatedPage<Goods>) goodsRepository.search(queryBuilder.build());
// 聚合结果
List<Brand> brands = getBrandAgg(brandAggsName,goodsResult);
List<Category> categories = getCategoryAgg(categoryAggsName,goodsResult);
/*
- 当分类聚合结果为1个统计规格参数
- 根据分类查询当前分类的搜索的规格参数
- 创建NativeQueryBuilder,使用上面搜索一样的条件
- 循环上面可搜索规格参数,依次添加聚合
- 处理结果k:参数名,options:聚合的结果数组
*/
List<Map<String,Object>> specs = null;
if (categories.size()==1){
specs = getSpecs(categories.get(0).getId(),basicQuery);
}
//计算总页数
long total = goodsResult.getTotalElements();
long totalPages = goodsResult.getTotalPages(); //(total + size - 1) / size;
return new SearchResult(total, totalPages, goodsResult.getContent(), categories, brands, specs);
}
//这个方法用来构建查询条件以及过滤条件
private QueryBuilder buildBasicQueryWithFilter(SearchRequest searchRequest) {
//构造布尔查询
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.must(QueryBuilders.matchQuery("all",searchRequest.getKey()));
//给这个查询加过滤
// 过滤条件构建器
BoolQueryBuilder filterQueryBuilder = QueryBuilders.boolQuery();
//取出map中的实体
for (Map.Entry<String, String> entry : searchRequest.getFilter().entrySet())
{
String key = entry.getKey();
String value = entry.getValue();
// 商品分类和品牌不用前后加修饰
if (key != "cid3" && key != "brandId") {
key = "specs." + key + ".keyword";
}
// 字符串类型,进行term查询
filterQueryBuilder.must(QueryBuilders.termQuery(key, value));
}
return queryBuilder.filter(filterQueryBuilder);
}
// 品牌
private List<Brand> getBrandAgg(String aggName, AggregatedPage<Goods>
result){
try {
LongTerms longTerms1 = (LongTerms) result.getAggregation(aggName);
List<Long> brandIds = new ArrayList<>();
for (LongTerms.Bucket bucket : longTerms1.getBuckets()) {
brandIds.add(bucket.getKeyAsNumber().longValue());
}
// return this.brandClient.queryBrandByIds(brandIds);
return this.brandClient.selectBrandByIds(brandIds);
} catch (Exception e) {
//logger.error("解析品牌数据错误:{}",e);
e.printStackTrace();
}
return null;
}
// 分类
private List<Category> getCategoryAgg(String categoryAggsName, AggregatedPage<Goods> goodsResult) {
LongTerms longTerms = (LongTerms)
goodsResult.getAggregation(categoryAggsName);
List<Long> categoryIds = new ArrayList<>();
for (LongTerms.Bucket bucket : longTerms.getBuckets()) {
categoryIds.add(bucket.getKeyAsNumber().longValue());
}
List<String> names = this.categoryClient.queryNameByIds(categoryIds);
List<Category> categories = new ArrayList<>();
for (int i = 0; i < names.size(); i++) {
Category category =new Category();
category.setId(categoryIds.get(i));
// category.setName(names.get(i));
category.setTitle(names.get(i));
categories.add(category);
}
return categories;
}
//规格参数的聚合应该和查询关联
private List<Map<String, Object>> getSpecs(Long cid, QueryBuilder query) {
List<Map<String,Object>> specs = new ArrayList<>();
// List<SpecParam> specParams =this.specificationClient.querySpecParam(null, cid, true, null);
SpecParam sp = new SpecParam();
sp.setCid(cid);
sp.setSearching(true);
List<SpecParam> specParams =
this.specificationClient.selectSpecParamApi(sp);
NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
//在做聚合之前先做查询,只有符合条件的规格参数才应该被查出来
queryBuilder.withQuery(query);
for (SpecParam specParam : specParams) {
String name = specParam.getName();//操作系统,cpu核数
queryBuilder.addAggregation(AggregationBuilders.terms(name).field("specs."+name +".keyword"));
}
AggregatedPage<Goods> aggs = (AggregatedPage<Goods>)
this.goodsRepository.search(queryBuilder.build());
Map<String, Aggregation> stringAggregationMap =
aggs.getAggregations().asMap();
for (SpecParam specParam : specParams) {
Map<String,Object> spec = new HashMap<>();
String name = specParam.getName();
if (stringAggregationMap.get(name) instanceof StringTerms) {
StringTerms stringTerms = (StringTerms) stringAggregationMap.get(name);
List<String> val = new ArrayList<>();
for (StringTerms.Bucket bucket : stringTerms.getBuckets()) {
val.add(bucket.getKeyAsString());
}
spec.put("k",name);//内存,存储空间,屏幕尺寸
spec.put("options",val);
specs.add(spec);
}
}
return specs;
}
}