springboot整合Elasticsearch-Rest-Client
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.5</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.atlinxi.gulimall</groupId>
<artifactId>gulimall-search</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>gulimall-search</name>
<description>elasticsearch检索服务</description>
<properties>
<java.version>1.8</java.version>
<elasticsearch.version>7.4.2</elasticsearch.version>
<spring-cloud.version>2020.0.4</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>com.atlinxi.gulimall</groupId>
<artifactId>gulimall-common</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.4.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
spring.cloud.nacos.discovery.server-addr=127.0.0.1:8848
spring.application.name=gulimall-search
package com.atlinxi.gulimall.search.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* es配置类,给容器中注入一个RestHighLevelClient
*/
@Configuration
public class GulimallElasticSearchConfig {
// 后端访问es的时候,出于安全考虑,可以携带一个请求头
// 现在暂时不用
public static final RequestOptions COMMON_OPTIONS;
static {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
// builder.addHeader("Authorization", "Bearer " + TOKEN);
// builder.setHttpAsyncResponseConsumerFactory(
// new HttpAsyncResponseConsumerFactory
// .HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
COMMON_OPTIONS = builder.build();
}
@Bean
public RestHighLevelClient esRestClient(){
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("192.168.56.10", 9200, "http")
// new HttpHost("localhost", 9201, "http")
));
return client;
}
}
package com.atlinxi.gulimall.search;
import com.alibaba.fastjson.JSON;
import com.atlinxi.gulimall.search.config.GulimallElasticSearchConfig;
import lombok.Data;
import lombok.ToString;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.Avg;
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
@SpringBootTest
class GulimallSearchApplicationTests {
@Autowired
RestHighLevelClient restHighLevelClient;
@Test
public void searchData()throws IOException{
// 1. 创建检索请求
SearchRequest searchRequest = new SearchRequest();
// 1.1 指定索引
searchRequest.indices("bank");
// 指定DSL,检索条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 1.2 构造检索条件
// 所有的函数名都对应原生es DSL语句
// searchSourceBuilder.query();
// searchSourceBuilder.from();
// searchSourceBuilder.size();
// searchSourceBuilder.aggregation();
searchSourceBuilder.query(QueryBuilders.matchQuery("address","mill"));
// 按照年龄的值分布进行聚合
TermsAggregationBuilder ageAgg = AggregationBuilders.terms("ageAgg").field("age").size(10);
searchSourceBuilder.aggregation(ageAgg);
// 计算平均薪资
AvgAggregationBuilder balanceAvg = AggregationBuilders.avg("balanceAvg").field("balance");
searchSourceBuilder.aggregation(balanceAvg);
// System.out.println(searchSourceBuilder.toString());
searchRequest.source(searchSourceBuilder);
// 2. 执行检索
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
// 3. 分析结果 searchResponse
// System.out.println(searchResponse.toString());
// 3.1 获取所有查到的数据
SearchHits hits = searchResponse.getHits();
SearchHit[] searchHits = hits.getHits();
for (SearchHit searchHit : searchHits) {
// searchHit.getIndex();
String string = searchHit.getSourceAsString();
Account account = JSON.parseObject(string, Account.class);
// accountGulimallSearchApplicationTests.Account(account_number=970, balance=19648, firstname=Forbes, lastname=Wallace, age=28, gender=M, address=990 Mill Road, employer=Pheast, email=forbeswallace@pheast.com, city=Lopezo, state=AK)
// accountGulimallSearchApplicationTests.Account(account_number=136, balance=45801, firstname=Winnie, lastname=Holland, age=38, gender=M, address=198 Mill Lane, employer=Neteria, email=winnieholland@neteria.com, city=Urie, state=IL)
// accountGulimallSearchApplicationTests.Account(account_number=345, balance=9812, firstname=Parker, lastname=Hines, age=38, gender=M, address=715 Mill Avenue, employer=Baluba, email=parkerhines@baluba.com, city=Blackgum, state=KY)
// accountGulimallSearchApplicationTests.Account(account_number=472, balance=25571, firstname=Lee, lastname=Long, age=32, gender=F, address=288 Mill Street, employer=Comverges, email=leelong@comverges.com, city=Movico, state=MT)
System.out.println("account" + account);
}
// 3.2 获取这次检索到的分析信息
Aggregations aggregations = searchResponse.getAggregations();
// for (Aggregation aggregation : aggregations.asList()) {
// System.out.println(aggregation.getName());
// }
Terms ageAgg1 = aggregations.get("ageAgg");
for (Terms.Bucket bucket : ageAgg1.getBuckets()) {
String keyAsString = bucket.getKeyAsString();
// 年龄38==>2
// 年龄28==>1
// 年龄32==>1
System.out.println("年龄" + keyAsString + "==>" + bucket.getDocCount());
}
Avg balanceAvg1 = aggregations.get("balanceAvg");
// 平均薪资:25208.0
System.out.println("平均薪资:" + balanceAvg1.getValue());
}
@ToString
@Data
static class Account{
private int account_number;
private int balance;
private String firstname;
private String lastname;
private int age;
private String gender;
private String address;
private String employer;
private String email;
private String city;
private String state;
}
/**
* 存储数据
*/
@Test
void indexData() throws IOException {
// 添加数据有多种方式,例如hashmap、直接将json粘在这儿
IndexRequest request = new IndexRequest("users");
request.id("1");
// request.source("userName","zhangsan","age",12,"gender","男");
// String jsonString = "{" +
// "\"user\":\"kimchy\"," +
// "\"postDate\":\"2013-01-30\"," +
// "\"message\":\"trying out Elasticsearch\"" +
// "}";
// request.source(jsonString, XContentType.JSON);
User user = new User();
user.setUserName("zs");
user.setAge(12);
user.setGender("man");
String jsonString = JSON.toJSONString(user);
request.source(jsonString, XContentType.JSON);
// 执行保存/更新操作
IndexResponse index = restHighLevelClient.index(request, GulimallElasticSearchConfig.COMMON_OPTIONS);
// 提取有用的响应数据
System.out.println(index);
}
@Data
class User{
private String userName;
private String gender;
private Integer age;
}
@Test
void contextLoads() {
System.out.println(restHighLevelClient);
}
}
sku在es中的存储模型分析
这样的结构比较容易检索
,但是会产生冗余字段
,
多个sku是共享一个spu的,所以以下结构中多个sku的attrs
是冗余的,
假设我们有100万个商品,平均属性有20个(2kb),那么冗余数据有2000m,也就是2g,那么整个商城的商品也就多了2g,加一个内存条就ok了
{
skuId:1,
skuTitle:华为XX,
price:998,
saleCount:99,
attrs:[
{尺寸:5寸},
{cpu:高通},
{分辨率:高清}
]
}
这样的结构,attrs只存了一次,没有了冗余数据,也容易检索
但是这样会有一个极大的问题,
搜索小米
,会出来粮食,手机,电器(很多sku都会包含小米)
,会检索出这些spu所有的规格属性,
假设小米的商品有10000个,涉及到4000个spu,4000个spu对应的所有属性,
esClient查询属性的时候,需要携带4000个spuId,因为是Long型数据,每一个都占8个字节,那么查询一次属性需要传递32kb的数据,
假设并发是1万,数据则是320m,如果面对百万并发,则是32G
最终面对百万并发,光传输4000个spuId,数据就高达32G,其他不说,光网络阻塞就非常长
sku索引{
skuId:1,
spuId:11,
xxxxx
}
attr索引{
spuId:11,
attrs:[
{尺寸:5寸},
{cpu:高通},
{分辨率:高清}
]
}
总之一句话,空间和时间
总是不能二者兼得,第一种浪费空间节省时间,第二种浪费时间节省空间,最终我们选择的是时间
PUT product
{
"mappings": {
"properties": {
"skuId": {
"type": "long"
},
"spuId": {
"type": "keyword"
},
"skuTitle": {
"type": "text",
"analyzer": "ik_smart"
},
"skuPrice": {
"type": "keyword"
},
"skuImg": {
"type": "keyword",
"index": false,
"doc_values": false
},
"saleCount": {
"type": "long"
},
"hasStock": {
"type": "boolean"
},
"hotScore": {
"type": "long"
},
"brandId": {
"type": "long"
},
"catalogId": {
"type": "long"
},
"brandName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"brandImg": {
"type": "keyword",
"index": false,
"doc_values": false
},
"catalogName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"attrs": {
"type": "nested",
"properties": {
"attrId": {
"type": "long"
},
"attrName": {
"type": "keyword",
"index": false,
"doc_values": false
},
"attrValue": {
"type": "keyword"
}
}
}
}
}
}
商品上架
spuInfoServiceImpl
/**
* 商品上架
* @param spuId
*/
@Transactional
@Override
public void up(Long spuId) {
// 1. 组装需要的数据
// 1.1 查出当前spuId对应的所有sku信息
List<SkuInfoEntity> skus = skuInfoService.getSkusBySpuId(spuId);
List<Long> skuIdList = skus.stream().map(SkuInfoEntity::getSkuId).collect(Collectors.toList());
// 4. 查询当前sku可以被检索的所有规格属性
List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrlistforspu(spuId);
List<Long> attrIds = baseAttrs.stream().map(attr -> {
return attr.getAttrId();
}).collect(Collectors.toList());
List<Long> searchAttrIds = attrService.selectSearchAttrs(attrIds);
Set<Long> idSet = new HashSet<>(searchAttrIds);
List<SkuEsModel.Attrs> attrsList = baseAttrs.stream().filter(item -> {
return idSet.contains(item.getAttrId());
}).map(item -> {
SkuEsModel.Attrs attrs1 = new SkuEsModel.Attrs();
BeanUtils.copyProperties(item, attrs1);
return attrs1;
}
).collect(Collectors.toList());
Map<Long, Boolean> stockMap = null;
try {
// 1. 发送远程调用,库存系统查询是否有库存
R skusHasStock = wareFeignService.getSkusHasStock(skuIdList);
// TypeReference 权限修饰符是 protected,所以我们要写成内部类的方式
List<SkuHasStockVo> data = skusHasStock.getData("data",new TypeReference<List<SkuHasStockVo>>(){});
stockMap = data.stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));
}catch (Exception e){
log.error("库存服务查询异常:",e);
}
// 1.2 封装每个sku的信息
Map<Long, Boolean> finalstockMap = stockMap;
List<SkuEsModel> upProducts = skus.stream().map(
sku -> {
SkuEsModel skuEsModel = new SkuEsModel();
BeanUtils.copyProperties(sku, skuEsModel);
// skuPrice,skuImg,hasStock,hotScore,brandName,brandImg,catalogName
// Attrs attrId,attrName,attrValue
skuEsModel.setSkuPrice(sku.getPrice());
skuEsModel.setSkuImg(sku.getSkuDefaultImg());
// 设置库存信息
skuEsModel.setHasStock(finalstockMap == null ? true : finalstockMap.get(sku.getSkuId()));
// todo 2. 热度评分 先默认设置0 但实际情况应该是后台可控的,比较复杂的操作
skuEsModel.setHotScore(0L);
// 3. 查询品牌和分类的名字信息
BrandEntity brand = brandService.getById(skuEsModel.getBrandId());
skuEsModel.setBrandName(brand.getName());
skuEsModel.setBrandImg(brand.getLogo());
CategoryEntity categoryEntity = categoryService.getById(skuEsModel.getCatalogId());
skuEsModel.setCatalogName(categoryEntity.getName());
// 设置检索属性
skuEsModel.setAttrs(attrsList);
return skuEsModel;
}
).collect(Collectors.toList());
// todo 5. 将数据发送给es进行保存 gulimall-search
R r = searchFeignService.productStatusUp(upProducts);
// 远程调用成功
if (r.getCode()==0){
// 修改当前spu状态
baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode());
}else {
// todo 重复调用,接口幂等性,重试机制
}
}
spuInfoServiceImpl是总的流程,下面都是它调用到的函数
比较简单的函数
// SkuInfoServiceImpl
public List<SkuInfoEntity> getSkusBySpuId(Long spuId) {
List<SkuInfoEntity> list = this.list(new QueryWrapper<SkuInfoEntity>().eq("spu_id",spuId));
return list;
}
// ProductAttrValueServiceImpl
public List<ProductAttrValueEntity> baseAttrlistforspu(Long spuId) {
List<ProductAttrValueEntity> entities = this.baseMapper.selectList(new QueryWrapper<ProductAttrValueEntity>().eq("spu_id", spuId));
return entities;
}
// AttrServiceImpl
@Override
public List<Long> selectSearchAttrs(List<Long> attrIds) {
return baseMapper.selectSearchAttrIds(attrIds);
}
<select id="selectSearchAttrIds" resultType="java.lang.Long">
select attr_id from pms_attr where attr_id in
<foreach collection="attrIds" separator="," item="id" open="(" close=")">
#{id}
</foreach>
and search_type = 1
</select>
// WareSkuServiceImpl
public List<SkuHasStockVo> getSkusHasStock(List<Long> skuIds) {
List<SkuHasStockVo> collect = skuIds.stream().map(skuId -> {
SkuHasStockVo vo = new SkuHasStockVo();
// 查询当前sku的总库存量
Long count = this.baseMapper.getSkuStock(skuId);
vo.setSkuId(skuId);
vo.setHasStock(count==null?false:count>0);
return vo;
}).collect(Collectors.toList());
return collect;
}
// SpuInfoDao.xml
<update id="updateSpuStatus">
update pms_spu_info set publish_status=#{code},update_time=NOW() where id = #{spuId}
</update>
search模块
package com.atlinxi.gulimall.search.controller;
import com.atlinxi.common.exception.BizCodeEnume;
import com.atlinxi.common.to.es.SkuEsModel;
import com.atlinxi.common.utils.R;
import com.atlinxi.gulimall.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.List;
@Slf4j
@RestController
@RequestMapping("/search/save")
public class ElasticSaveController {
@Autowired
private ProductSaveService productSaveService;
/**
* 上架商品
*/
@PostMapping("/product")
public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModels){
boolean b = false;
try {
b = productSaveService.productStatusUp(skuEsModels);
} catch (IOException e) {
log.error("ElasticSaveController商品上架错误:{}",e);
return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.PRODUCT_UP_EXCEPTION.getMessage());
}
if (!b){
return R.ok();
}else {
return R.error();
}
}
}
package com.atlinxi.gulimall.search.service.impl;
import com.alibaba.fastjson.JSON;
import com.atlinxi.common.to.es.SkuEsModel;
import com.atlinxi.gulimall.search.config.GulimallElasticSearchConfig;
import com.atlinxi.gulimall.search.constant.EsConstant;
import com.atlinxi.gulimall.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@Service
@Slf4j
public class ProductSaveServiceImpl implements ProductSaveService {
@Autowired
private RestHighLevelClient restHighLevelClient;
/**
*
* @param skuEsModels
* @return
*/
@Override
public boolean productStatusUp(List<SkuEsModel> skuEsModels) throws IOException {
// 保存到es
// 1. 建立索引 product 建立好映射关系(kibana操作)
// 2. 给es中保存这些数据
BulkRequest bulkRequest = new BulkRequest();
for (SkuEsModel skuEsModel : skuEsModels) {
IndexRequest indexRequest = new IndexRequest(EsConstant.Product_INDEX);
indexRequest.id(skuEsModel.getSkuId().toString());
String s = JSON.toJSONString(skuEsModel);
indexRequest.source(s, XContentType.JSON);
bulkRequest.add(indexRequest);
}
BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
// todo 如果批量错误,处理错误
boolean b = bulk.hasFailures();
List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {
return item.getId();
}).collect(Collectors.toList());
log.info("商品上架成功:{}",collect);
return b;
}
}
封装数据的类
/**
* Copyright (c) 2016-2019 人人开源 All rights reserved.
*
* https://www.renren.io
*
* 版权所有,侵权必究!
*/
package com.atlinxi.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import org.apache.http.HttpStatus;
import java.util.HashMap;
import java.util.Map;
/**
* 返回数据
*
* @author Mark sunlightcs@gmail.com
*
*
* 老师是用泛型的方式封装的data,
* 在feign接口中返回泛型类时,由于java的泛型机制,在实例化之前无法得到具体的类型 ,
* 因此,虽然服务提供方返回的是具体实例的数据,但是在客户端decode时,无法转化为具体的类。
*
* 上面的话看不太懂,翻译成人话就是,feign在被远程调用返回结果的时候,泛型是null
*
* 因为R继承了HashMap,我们写的所有私有属性都没用,只能存键值对,具体原因未知,
*
* public class R<T> extends HashMap<String, Object> {
* private static final long serialVersionUID = 1L;
*
* private T data;
*
* public T getData() {
* return this.data;
* }
*
*
* public void setData(T data) {
* this.data = data;
* }
*/
public class R extends HashMap<String, Object> {
private static final long serialVersionUID = 1L;
public R setData(Object data){
put("data",data);
return this;
}
//利用fastjson进行反序列化
public <T> T getData(TypeReference<T> typeReference) {
Object data = get("data"); //默认是map
String jsonString = JSON.toJSONString(data);
T t = JSON.parseObject(jsonString, typeReference);
return t;
}
//利用fastjson进行反序列化
public <T> T getData(String key,TypeReference<T> typeReference) {
Object data = get(key); //默认是map
String jsonString = JSON.toJSONString(data);
T t = JSON.parseObject(jsonString, typeReference);
return t;
}
public R() {
put("code", 0);
put("msg", "success");
}
public static R error() {
return error(HttpStatus.SC_INTERNAL_SERVER_ERROR, "未知异常,请联系管理员");
}
public static R error(String msg) {
return error(HttpStatus.SC_INTERNAL_SERVER_ERROR, msg);
}
public static R error(int code, String msg) {
R r = new R();
r.put("code", code);
r.put("msg", msg);
return r;
}
public static R ok(String msg) {
R r = new R();
r.put("msg", msg);
return r;
}
public static R ok(Map<String, Object> map) {
R r = new R();
r.putAll(map);
return r;
}
public static R ok() {
return new R();
}
public R put(String key, Object value) {
super.put(key, value);
return this;
}
public int getCode(){
return (Integer) this.get("code");
}
}
枚举
package com.atlinxi.common.constant;
public class ProductConstant {
public enum AttrEnum{
ATTR_TYPE_BASE(1,"基本属性"),ATTR_TYPE_SALE(0,"销售属性");
private int code;
private String msg;
AttrEnum(int code, String msg) {
this.code = code;
this.msg = msg;
}
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
}
public enum StatusEnum{
NEW_SPU(0,"新建"),SPU_UP(1,"商品上架"),
SPU_DOWN(2,"商品下架");
private int code;
private String msg;
StatusEnum(int code, String msg) {
this.code = code;
this.msg = msg;
}
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
}
}
但是她总觉得怪怪的,李国华的眼睛里有一种研究的意味。很久以后,伊纹才会知道,李国华想要在她脸上预习思琪将来的表情。
房思琪的初恋乐园
林奕含