P109测试数据
地址:
https://raw.githubusercontent.com/elastic/elasticsearch/7.4/docs/src/test/resources/accounts.json
POST bank/account/_bulk
安装分词ki
yum install wget
mkdir /mydata/elasticsearch/plugins/ik
cd /mydata/elasticsearch/plugins/ik
下载ki v7.4.2 [记得在ik目录下]
wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.4.2/elasticsearch-analysis-ik-7.4.2.zip
安装unzip
yum install -y unzip zip
解压
unzip elasticsearch-analysis-ik-7.4.2.zip
解压完后记得删除压缩包,以免报错
rm -rf *.zip
查看是否安装成功
重启并测试
docker restart elasticsearch
刷新kibana页面,开始测试
POST _analyze
{
"analyzer":"ik_max_word",
"text":"我是中国人"
}
自定义扩展词库
启动Nginx
docker run -p 80:80 --name nginx \
> -v /mydata/nginx/html:/usr/share/nginx/html \
> -v /mydata/nginx/logs:/var/log/nginx \
> -v /mydata/nginx/conf:/etc/nginx \
> -d nginx:1.10
index.html中输入gulimall,测试
cd es/
vi fenci.txt
自己输入自定义的词库
测试--乱码了哈哈哈
cd /mydata/elasticsearch/plugins/ik/
cd config/
ls
vi IKAnalyzer.cfg.xml
docker r模块estart elasticsearch
测试数据
PUT product
{
"mappings": {
"properties": {
"skuId": {
"type": "long"
},
"spuId": {
"type": "keyword"
},
"skuTitle": {
"type": "text",
"analyzer": "ik_smart"
},
"skuPrice": {
"type": "keyword"
},
"skuImg": {
"type": "keyword",
"index": false,
"doc_values": false
},
"saleCount": {
"type": "long"
},
"hasStock": {
"type": "boolean"
},
"hotScore": {
"type": "long"
},
"brandId": {
"type": "long"
},
"catalogId": {
"type": "long"
},
"brandName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"brandImg": {
"type": "keyword",
"index": false,
"doc_values": false
},
"catalogName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"attrs": {
"type": "nested",
"properties": {
"attrId": {
"type": "long"
},
"attrName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"attrValue": {
"type": "keyword"
}
}
}
}
}
}
springboot整合elasticsearch
<elasticsearch.version>7.4.2</elasticsearch.version>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
<exclusions>
<exclusion>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
</exclusion>
<exclusion>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.4.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.4.2</version>
</dependency>
@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
package com.example.search.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class GulimallElasticSearchConfig {
public static final RequestOptions COMMON_OPTIONS;
static {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
COMMON_OPTIONS = builder.build();
}
@Bean
public RestHighLevelClient esRestClient(){
RestClientBuilder builder=null;
//final String hostname, final int port, final String scheme
builder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));
RestHighLevelClient client = new RestHighLevelClient(builder);
// RestHighLevelClient client = new RestHighLevelClient(
// RestClient.builder(new HttpHost("192.168.56.10",9200,"http"))
// );
return client;
}
}
package com.example.search;
import com.alibaba.fastjson.JSON;
import com.example.search.config.GulimallElasticSearchConfig;
import lombok.Data;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
@SpringBootTest
class GulimallSearchApplicationTests {
@Autowired
private RestHighLevelClient client;
/**
* 测试存储数据到es
*/
@Test
void indexData() throws IOException {
IndexRequest indexRequest = new IndexRequest("users");
indexRequest.id("1"); //数据的id
User user = new User();
user.setUserName("zhang");
user.setAge(18);
user.setGender("男");
String s = JSON.toJSONString(user);
indexRequest.source(s,XContentType.JSON);
//执行操作
IndexResponse index = client.index(indexRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
System.out.println(index);
}
@Data
class User{
private String userName;
private String gender;
private Integer age;
}
@Test
void contextLoads() {
System.out.println(client);
}
}
测试复杂检索
@Data
@ToString
@AllArgsConstructor
@NoArgsConstructor
static class Account{
private int account_number;
private int balance;
private String firstname;
private String lastname;
private int age;
private String gender;
private String address;
private String employer;
private String email;
private String city;
private String state;
}
@Autowired
private RestHighLevelClient client;
@Test
public void searchData() throws IOException {
//1.创建检索请求
SearchRequest searchRequest = new SearchRequest();
//指定索引
searchRequest.indices("bank");
//指定DSL,检索条件
//SearchSourceBuilder sourceBuilder封装的条件
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
searchRequest.source(sourceBuilder);
//1.1构造检索条件
sourceBuilder.query(QueryBuilders.matchQuery("address","mill"));
//1.2按年龄的值分布聚合
TermsAggregationBuilder ageAgg =AggregationBuilders.terms("ageAgg").field("age").size(10);
sourceBuilder.aggregation(ageAgg);
//1.3计算平均分布聚合
AvgAggregationBuilder balanceAvg = AggregationBuilders.avg("balanceAvg").field("balance");
sourceBuilder.aggregation(balanceAvg);
System.out.println("检索条件"+sourceBuilder.toString());
searchRequest.source(sourceBuilder);
//2.执行检索
SearchResponse searchResponse = client.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
//3.分析结果
System.out.println(searchResponse.toString());
//3.1.获取
//外边的大hit
SearchHits hits = searchResponse.getHits();
//大hit里面的小hit
SearchHit[] searchHits = hits.getHits();//真正命中的所有记录
for(SearchHit hit : searchHits){
String sourceAsString = hit.getSourceAsString();
Account account = JSON.parseObject(sourceAsString, Account.class);
System.out.println("account "+account);
}
//3.2.获取这次检索到的分析信息
Aggregations aggregations = searchResponse.getAggregations();
Terms ageAgg1 = aggregations.get("ageAgg");
for(Terms.Bucket bucket: ageAgg1.getBuckets()){
String keyAsString = bucket.getKeyAsString();
System.out.println("年龄"+keyAsString+"===="+bucket.getDocCount());
}
Avg balanceAvg1 = aggregations.get("balanceAvg");
System.out.println("平均薪资"+balanceAvg1.getValue());
}
SPU(Standard Product Unit):标准化产品单元。是商品信息聚合的最小单位,是一组可复用、易检索的标准化信息的集合,该集合描述了一个产品的特性。
SKU=Stock Keeping Unit(库存量单位)。即库存进出计量的基本单元,可以是以件,盒,托盘等为单位。SKU这是对于大型连锁超市DC(配送中心)物流管理的一个必要的方法。现在已经被引申为产品统一编号的简称,每种产品均对应有唯一的SKU号。
比如,咱们购买一台iPhoneX手机,iPhoneX手机就是一个SPU,但是你购买的时候,不可能是以iPhoneX手机为单位买的,商家也不可能以iPhoneX为单位记录库存。必须要以什么颜色什么版本的iPhoneX为单位。比如,你购买的是一台银色、128G内存的、支持联通网络的iPhoneX ,商家也会以这个单位来记录库存数。那这个更细致的单位就叫库存单元(SKU)。
商品上架
SpuInfoController
/**
* 商品上架
*/
@PostMapping("/{spuId}/up")
public R spuUp(@PathVariable("spuId")Long spuId){
spuInfoService.up(spuId);
return R.ok();
}
package com.example.common.es;
import lombok.Data;
import java.math.BigDecimal;
import java.util.List;
@Data
public class SkuEsModel {
private Long skuId;
private Long spuId;
private String skuTitle;
private BigDecimal skuPrice;
private String skuImg;
private Long saleCount;
private Boolean hasStock;
private Long hotScore;
private Long brandId;
private Long catalogId;
private String brandName;
private String brandImg;
private String catalogName;
private List<Attrs> attrs;
@Data
public static class Attrs{
private String attrId;
private String attrName;
private String attrValue;
}
}
SkuInfoServiceImpl---当前spuid对应的所有sku信息,品牌的名字
@Override
public List<SkuInfoEntity> getSkusBySpuId(Long spuId) {
List<SkuInfoEntity> list = this.list(new QueryWrapper<SkuInfoEntity>().eq("spu_id", spuId));
return list;
}
ProductAttrValueServiceImpl--当前spu对应的的所有attr信息
@Override
public List<ProductAttrValueEntity> baseAttrListForSpu(Long spuId) {
List<ProductAttrValueEntity> entities = this.baseMapper.selectList(new QueryWrapper<ProductAttrValueEntity>().eq("spu_id", spuId));
return entities;
}
AttrServiceImpl--在指定的所有属性集合里边,找出检索属性放到idSet
@Override
public List<Long> selectSearchAttrIds(List<Long> attrIds) {
return baseMapper.selectSearvhAttrIds(attrIds);
}
product模块--发送远程调用,库存系统查询是否有库存
@FeignClient("gulimall-ware")
public interface WareFeignService {
@PostMapping("/ware/waresku/hasstock")
R getSkusHasStock(@RequestBody List<Long> skuIds);
}
ware模块--库存系统查询是否有库存
/**
* 查询sku是否有库存
* @param skuIds
* @return
*/
@PostMapping("/hasstock")
public R getSkusHasStock(@RequestBody List<Long> skuIds){
//返回sku_id,当前库存量stock
List<SkuHasStockVo>vos = wareSkuService.getSkusHasSock(skuIds);
return R.ok().setData(vos);
// return R.ok().put("data",vos);
}
WareSkuServiceImpl
//检查每一个商品的库存
@Override
public List<SkuHasStockVo> getSkusHasSock(List<Long> skuIds) {
List<SkuHasStockVo> collect = skuIds.stream().map(skuId -> {
SkuHasStockVo vo = new SkuHasStockVo();
//查询当前sku的总库存 wms_ware_sku
Long count = baseMapper.getSkuStock(skuId);
vo.setSkuId(skuId);
vo.setHasStock(count==null?false:count>0);
return vo;
}).collect(Collectors.toList());
return collect;
}
product模块--数据发送给es进行保存:(负责保存gulimall-search)
@FeignClient("gulimall-search")
public interface SearchFeignSerice {
//上架商品
@PostMapping("/search/save/product")
R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes);
}
package com.example.search.controller;
import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {
@Autowired
ProductSaveService productSaveService;
//上架商品
@PostMapping("/product")
public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {
boolean b=false; //
try{
b = productSaveService.productStatusUp(skuEsModes);
}catch (Exception e){
log.error("ElasticSaveController商品上架错误:{}",e);
return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());
}
if(!b){ return R.ok();}
else{
//有错误
return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}
}
}
search--
@Service
public class ProductSaveServiceImpl implements ProductSaveService {
@Autowired
RestHighLevelClient restHighLevelClient;
@Override
public boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {
//保存到es
//1.给es中建立索引。product,建立好映射关系
BulkRequest bulkRequest = new BulkRequest();
for(SkuEsModel model:skuEsModes){
//构造保存请求
IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);
indexRequest.id(model.getSkuId().toString());
String s = JSON.toJSONString(model);
indexRequest.source(s, XContentType.JSON);
bulkRequest.add(indexRequest);//
}
//2.给es中保存这些数据
//批量操作sku数据
BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
//统计上架错误的sku
//TODO 如果批量错误
boolean b = bulk.hasFailures();
List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {
return item.getId();
}).collect(Collectors.toList());
log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());
return b;
}
}
package com.example.search.controller;
import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {
@Autowired
ProductSaveService productSaveService;
//上架商品
@PostMapping("/product")
public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {
boolean b=false; //
try{
b = productSaveService.productStatusUp(skuEsModes);
}catch (Exception e){
log.error("ElasticSaveController商品上架错误:{}",e);
return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());
}
if(!b){ return R.ok();}
else{
//有错误
return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}
}
}
package com.example.search.constant;
public class EsConstant {
public static final String PRODUCT_INDEX="product";//sku数据在es中的索引
}
package com.example.search.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class GulimallElasticSearchConfig {
public static final RequestOptions COMMON_OPTIONS;
static {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
COMMON_OPTIONS = builder.build();
}
@Bean
public RestHighLevelClient esRestClient(){
RestClientBuilder builder=null;
//final String hostname, final int port, final String scheme
builder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));
RestHighLevelClient client = new RestHighLevelClient(builder);
return client;
}
}
public class ProductSaveServiceIm
@Slf4j
@Service
public class ProductSaveServiceImpl implements ProductSaveService {
@Autowired
RestHighLevelClient restHighLevelClient;
@Override
public boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {
//保存到es
//1.给es中建立索引。product,建立好映射关系
BulkRequest bulkRequest = new BulkRequest();
for(SkuEsModel model:skuEsModes){
//构造保存请求
IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);
indexRequest.id(model.getSkuId().toString());
String s = JSON.toJSONString(model);
indexRequest.source(s, XContentType.JSON);
bulkRequest.add(indexRequest);//
}
//2.给es中保存这些数据
//批量操作sku数据
BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
//统计上架错误的sku
//TODO 如果批量错误
boolean b = bulk.hasFailures();
List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {
return item.getId();
}).collect(Collectors.toList());
log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());
return b;
}
}
SpuInfoServiceImpl
@Override
public void up(Long spuId) {
//1.查出当前spuid对应的所有sku信息,品牌的名字
List<SkuInfoEntity>skus = skuInfoService.getSkusBySpuId(spuId);
//TODO 4.查出当前sku所有可以被用来检索的规格属性信息pms_product_attr_value
//获取当前spu对应的的所有attr信息
List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrListForSpu(spuId);
//挑出所有检索的信息
List<Long> attrIds = baseAttrs.stream().map((attr -> {
return attr.getAttrId();
})).collect(Collectors.toList());
//pms_attr,在指定的所有属性集合里边,找出检索属性idSet
List<Long> searchAttrIds = attrService.selectSearchAttrIds(attrIds);
Set<Long> idSet = new HashSet<>(searchAttrIds);
// Set<Long> idSet = searchAttrIds.stream().collect(Collectors.toSet());
List<SkuEsModel.Attrs> attrsList = baseAttrs.stream().filter(item -> {
return idSet.contains(item.getAttrId());
}).map(item -> {
SkuEsModel.Attrs attrs1 = new SkuEsModel.Attrs();
BeanUtils.copyProperties(item, attrs1);
return attrs1;
}).collect(Collectors.toList());
//拿到所有sku的id
List<Long> skuIdList = skus.stream().map(SkuInfoEntity::getSkuId).collect(Collectors.toList());
//TODO 1。发送远程调用,库存系统查询是否有库存
Map<Long, Boolean> stockMap =null;
try{
R r = wareFeignService.getSkusHasStock(skuIdList);
// TypeReference<List<SkuHasStockVo>> typeReference = new TypeReference<List<SkuHasStockVo>>() {};
// stockMap = skusHasStock.getData(typeReference).stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));
//SkuHasStockVo::getSkuId --->key
TypeReference<List<SkuHasStockVo>> typeReference = new TypeReference<List<SkuHasStockVo>>(){};
stockMap = r.getData(typeReference).stream().collect(Collectors
.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));
}catch(Exception e){
log.error("库存查询异常:原因{}",e);
}
//2.封装每个sku的信息
Map<Long,Boolean> finalStockMap = stockMap;
List<SkuEsModel>upProducts= skus.stream().map(sku -> {
//组装需要的数据pms_sku_info
SkuEsModel esModel = new SkuEsModel();
BeanUtils.copyProperties(sku,esModel);
esModel.setSkuPrice(sku.getPrice());
esModel.setSkuImg(sku.getSkuDefaultImg());
//hasStokc,hotSore 设置库存信息
if(finalStockMap ==null){
esModel.setHasStock(true);
}else{
esModel.setHasStock(finalStockMap.get(sku.getSkuId()));
}
//TODO 2.热度评分 刚上架是0
esModel.setHotScore(0L);
//TODO 3.查询品牌和分类的名字信息
BrandEntity brand = brandService.getById(esModel.getBrandId());
esModel.setBrandName(brand.getName());
//esModel.setBrandId(brand.getBrandId());//
esModel.setBrandImg(brand.getLogo());
//分类信息
CategoryEntity category = categoryService.getById(esModel.getCatalogId());
// esModel.setCatalogId(category.getCatId());//
esModel.setCatalogName(category.getName());
//设置检索属性
esModel.setAttrs(attrsList);
// BeanUtils.copyProperties(sku,esModel);//zheli ?
return esModel;
}).collect(Collectors.toList());
//TODO 5.将数据发送给es进行保存:gulimall-search
R r = searchFeignSerice.productStatusUp(upProducts);
if(r.getCode()==0) {
//远程调用成功
//TODO 6.修改当前spu的状态 pms_spu_info
baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode());
}else{
//远程调用失败
// TODO 重复调用?;接口幂等性 重试机制
}
}
用于数据逆转--添加getData和setData
public class R extends HashMap<String, Object> {
private static final long serialVersionUID = 1L;
//利用fastJSON进行逆转
public<T> T getData(TypeReference<T> typeReference){
Object data= get("data");//默认是map类型
String s = JSON.toJSONString(data);
T t = JSON.parseObject("s",typeReference);
return t;
}
public R setData(Object data){
put("data",data);
return this;
}