day10谷粒商城

学习吧黑黑

已于 2022-10-28 15:26:46 修改

阅读量769

点赞数

文章标签： elasticsearch 搜索引擎大数据

于 2022-08-19 14:17:28 首次发布

本文链接：https://blog.csdn.net/qq_43960586/article/details/126401663

版权

P109测试数据

地址：

https://raw.githubusercontent.com/elastic/elasticsearch/7.4/docs/src/test/resources/accounts.json

POST bank/account/_bulk

安装分词ki

yum install wget

mkdir  /mydata/elasticsearch/plugins/ik

cd /mydata/elasticsearch/plugins/ik

下载ki v7.4.2 [记得在ik目录下]

wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.4.2/elasticsearch-analysis-ik-7.4.2.zip

安装unzip

yum install -y unzip zip

解压

 unzip  elasticsearch-analysis-ik-7.4.2.zip

解压完后记得删除压缩包，以免报错

 rm -rf *.zip

查看是否安装成功

重启并测试

docker restart elasticsearch

刷新kibana页面，开始测试

POST _analyze
{
  "analyzer":"ik_max_word",
  "text":"我是中国人"
}

自定义扩展词库

只是为了复制出配置而安装Nginx

启动Nginx

docker run -p 80:80 --name nginx \
> -v /mydata/nginx/html:/usr/share/nginx/html \
> -v /mydata/nginx/logs:/var/log/nginx \
> -v /mydata/nginx/conf:/etc/nginx \
> -d nginx:1.10

index.html中输入gulimall，测试

cd es/

vi fenci.txt

自己输入自定义的词库

测试--乱码了哈哈哈

cd /mydata/elasticsearch/plugins/ik/

cd config/

ls

vi IKAnalyzer.cfg.xml

docker r模块estart elasticsearch

测试数据

PUT product
{
  "mappings": {
    "properties": {
      "skuId": {
        "type": "long"
      },
      "spuId": {
        "type": "keyword"
      },
      "skuTitle": {
        "type": "text",
        "analyzer": "ik_smart"
      },
      "skuPrice": {
        "type": "keyword"
      },
      "skuImg": {
        "type": "keyword",
        "index": false,
        "doc_values": false
      },
      "saleCount": {
        "type": "long"
      },
      "hasStock": {
        "type": "boolean"
      },
      "hotScore": {
        "type": "long"
      },
      "brandId": {
        "type": "long"
      },
      "catalogId": {
        "type": "long"
      },
      "brandName": {
        "type": "keyword",
        "index": false,
        "doc_values": false
      },
      "brandImg": {
        "type": "keyword",
        "index": false,
        "doc_values": false
      },
      "catalogName": {
        "type": "keyword",
        "index": false,
        "doc_values": false
      },
      "attrs": {
        "type": "nested",
        "properties": {
          "attrId": {
            "type": "long"
          },
          "attrName": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "attrValue": {
            "type": "keyword"
          }
        }
      }
    }
  }
}

springboot整合elasticsearch

 <elasticsearch.version>7.4.2</elasticsearch.version>

 <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>${elasticsearch.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.elasticsearch</groupId>
                    <artifactId>elasticsearch</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.elasticsearch.client</groupId>
                    <artifactId>elasticsearch-rest-client</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client</artifactId>
            <version>7.4.2</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>7.4.2</version>
        </dependency>

@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)

package com.example.search.config;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class GulimallElasticSearchConfig {
    public static final RequestOptions COMMON_OPTIONS;
    static {
        RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
        COMMON_OPTIONS = builder.build();
    }

    @Bean
    public RestHighLevelClient esRestClient(){
        RestClientBuilder builder=null;
        //final String hostname, final int port, final String scheme
        builder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));
        RestHighLevelClient client = new RestHighLevelClient(builder);
//        RestHighLevelClient client = new RestHighLevelClient(
//                RestClient.builder(new HttpHost("192.168.56.10",9200,"http"))
//        );

        return client;
    }
}

package com.example.search;

import com.alibaba.fastjson.JSON;
import com.example.search.config.GulimallElasticSearchConfig;
import lombok.Data;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

import java.io.IOException;

@SpringBootTest
class GulimallSearchApplicationTests {
    @Autowired
private RestHighLevelClient client;

    /**
     * 测试存储数据到es
     */
    @Test
    void indexData() throws IOException {
        IndexRequest indexRequest = new IndexRequest("users");
        indexRequest.id("1"); //数据的id
        User user = new User();
        user.setUserName("zhang");
        user.setAge(18);
        user.setGender("男");
        String s = JSON.toJSONString(user);
        indexRequest.source(s,XContentType.JSON);
        //执行操作
        IndexResponse index = client.index(indexRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
        System.out.println(index);
    }
    @Data
class User{
        private String userName;
        private String gender;
        private Integer age;
}
    @Test
    void contextLoads() {
        System.out.println(client);
    }

}

测试复杂检索

 @Data
    @ToString
    @AllArgsConstructor
    @NoArgsConstructor
   static class Account{
       private int account_number;
       private int balance;
       private String firstname;
       private String lastname;
       private  int age;
       private String gender;
       private String address;
       private  String employer;
       private  String email;
       private  String city;
       private String state;
    }
    @Autowired
private RestHighLevelClient client;
    @Test
    public void searchData() throws IOException {
        //1.创建检索请求
        SearchRequest searchRequest = new SearchRequest();
        //指定索引
        searchRequest.indices("bank");
        //指定DSL，检索条件
        //SearchSourceBuilder sourceBuilder封装的条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        searchRequest.source(sourceBuilder);
        //1.1构造检索条件
        sourceBuilder.query(QueryBuilders.matchQuery("address","mill"));
        //1.2按年龄的值分布聚合
        TermsAggregationBuilder ageAgg =AggregationBuilders.terms("ageAgg").field("age").size(10);
        sourceBuilder.aggregation(ageAgg);
        //1.3计算平均分布聚合
         AvgAggregationBuilder balanceAvg = AggregationBuilders.avg("balanceAvg").field("balance");
        sourceBuilder.aggregation(balanceAvg);
        System.out.println("检索条件"+sourceBuilder.toString());
        searchRequest.source(sourceBuilder);
        //2.执行检索
         SearchResponse searchResponse = client.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
      //3.分析结果
        System.out.println(searchResponse.toString());
        //3.1.获取
        //外边的大hit
         SearchHits hits = searchResponse.getHits();
        //大hit里面的小hit
         SearchHit[] searchHits = hits.getHits();//真正命中的所有记录
        for(SearchHit hit : searchHits){
             String sourceAsString = hit.getSourceAsString();
             Account account = JSON.parseObject(sourceAsString, Account.class);
            System.out.println("account "+account);
        }
        //3.2.获取这次检索到的分析信息

        Aggregations aggregations = searchResponse.getAggregations();
        Terms ageAgg1 = aggregations.get("ageAgg");
        for(Terms.Bucket bucket: ageAgg1.getBuckets()){
            String keyAsString = bucket.getKeyAsString();
            System.out.println("年龄"+keyAsString+"===="+bucket.getDocCount());
        }
        Avg balanceAvg1 = aggregations.get("balanceAvg");
        System.out.println("平均薪资"+balanceAvg1.getValue());
    }

SPU(Standard Product Unit)：标准化产品单元。是商品信息聚合的最小单位，是一组可复用、易检索的标准化信息的集合，该集合描述了一个产品的特性。

SKU=Stock Keeping Unit（库存量单位）。即库存进出计量的基本单元，可以是以件，盒，托盘等为单位。SKU这是对于大型连锁超市DC（配送中心）物流管理的一个必要的方法。现在已经被引申为产品统一编号的简称，每种产品均对应有唯一的SKU号。

比如，咱们购买一台iPhoneX手机，iPhoneX手机就是一个SPU，但是你购买的时候，不可能是以iPhoneX手机为单位买的，商家也不可能以iPhoneX为单位记录库存。必须要以什么颜色什么版本的iPhoneX为单位。比如，你购买的是一台银色、128G内存的、支持联通网络的iPhoneX ，商家也会以这个单位来记录库存数。那这个更细致的单位就叫库存单元（SKU）。

商品上架

SpuInfoController

    /**
     * 商品上架
     */
@PostMapping("/{spuId}/up")
    public R spuUp(@PathVariable("spuId")Long spuId){
       spuInfoService.up(spuId);
        return R.ok();
    }

package com.example.common.es;

import lombok.Data;

import java.math.BigDecimal;
import java.util.List;

@Data
public class SkuEsModel {
    private Long skuId;
    private Long spuId;
    private String skuTitle;
    private BigDecimal skuPrice;
    private String skuImg;
    private Long saleCount;
    private Boolean hasStock;
    private Long hotScore;
    private Long brandId;
    private Long catalogId;
    private String brandName;
    private String brandImg;
    private String catalogName;
    private List<Attrs> attrs;
@Data
  public  static class Attrs{
        private String attrId;
        private String attrName;
        private String attrValue;
    }
}

SkuInfoServiceImpl---当前spuid对应的所有sku信息，品牌的名字

 @Override
    public List<SkuInfoEntity> getSkusBySpuId(Long spuId) {
         List<SkuInfoEntity> list = this.list(new QueryWrapper<SkuInfoEntity>().eq("spu_id", spuId));
        return list;
    }

ProductAttrValueServiceImpl--当前spu对应的的所有attr信息

@Override
    public List<ProductAttrValueEntity> baseAttrListForSpu(Long spuId) {
        List<ProductAttrValueEntity> entities = this.baseMapper.selectList(new QueryWrapper<ProductAttrValueEntity>().eq("spu_id", spuId));
        return entities;
    }

AttrServiceImpl--在指定的所有属性集合里边，找出检索属性放到idSet

 @Override
    public List<Long> selectSearchAttrIds(List<Long> attrIds) {
        return baseMapper.selectSearvhAttrIds(attrIds);
    }

product模块--发送远程调用，库存系统查询是否有库存

@FeignClient("gulimall-ware")
public interface WareFeignService {
    @PostMapping("/ware/waresku/hasstock")
 R getSkusHasStock(@RequestBody List<Long> skuIds);

}

ware模块--库存系统查询是否有库存

    /**
     * 查询sku是否有库存
     * @param skuIds
     * @return
     */
    @PostMapping("/hasstock")
    public R getSkusHasStock(@RequestBody List<Long> skuIds){
        //返回sku_id，当前库存量stock
      List<SkuHasStockVo>vos =   wareSkuService.getSkusHasSock(skuIds);
      return R.ok().setData(vos);
//        return R.ok().put("data",vos);
    }

WareSkuServiceImpl

//检查每一个商品的库存
    @Override
    public List<SkuHasStockVo> getSkusHasSock(List<Long> skuIds) {
         List<SkuHasStockVo> collect = skuIds.stream().map(skuId -> {
             SkuHasStockVo vo = new SkuHasStockVo();
            //查询当前sku的总库存 wms_ware_sku
            Long count = baseMapper.getSkuStock(skuId);
            vo.setSkuId(skuId);
            vo.setHasStock(count==null?false:count>0);
            return vo;
        }).collect(Collectors.toList());
        return collect;
    }

product模块--数据发送给es进行保存：（负责保存gulimall-search）

@FeignClient("gulimall-search")
public interface SearchFeignSerice {
    //上架商品
    @PostMapping("/search/save/product")
     R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes);
}

package com.example.search.controller;

import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {

   @Autowired
    ProductSaveService productSaveService;
    //上架商品
    @PostMapping("/product")
    public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {
   boolean b=false; //
    try{
        b = productSaveService.productStatusUp(skuEsModes);
    }catch (Exception e){
     log.error("ElasticSaveController商品上架错误:{}",e);
    return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());
    }
    if(!b){  return R.ok();}
   else{
       //有错误
       return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}
    }
}

search--

@Service
public class ProductSaveServiceImpl implements ProductSaveService {
   @Autowired
    RestHighLevelClient restHighLevelClient;

    @Override
    public boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {
        //保存到es
        //1.给es中建立索引。product，建立好映射关系
        BulkRequest bulkRequest = new BulkRequest();
        for(SkuEsModel model:skuEsModes){
            //构造保存请求
            IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);
            indexRequest.id(model.getSkuId().toString());
            String s = JSON.toJSONString(model);
            indexRequest.source(s, XContentType.JSON);

            bulkRequest.add(indexRequest);//
        }
       //2.给es中保存这些数据
        //批量操作sku数据
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
       //统计上架错误的sku
        //TODO  如果批量错误
        boolean b = bulk.hasFailures();
        List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {
            return item.getId();

        }).collect(Collectors.toList());
        log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());
        return b;
    }

}

package com.example.search.controller;

import com.example.common.exception.BizCodeEnume;
import com.example.common.to.es.SkuEsModel;
import com.example.common.utils.R;
import com.example.search.service.ProductSaveService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.List;
@Slf4j
@RequestMapping("/search/save")
@RestController
public class ElasticSaveController {

   @Autowired
    ProductSaveService productSaveService;
    //上架商品
    @PostMapping("/product")
    public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModes) throws IOException {
   boolean b=false; //
    try{
        b = productSaveService.productStatusUp(skuEsModes);
    }catch (Exception e){
     log.error("ElasticSaveController商品上架错误:{}",e);
    return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());
    }
    if(!b){  return R.ok();}
   else{
       //有错误
       return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.VALID_EXCEPTION.getMsg());}
    }
}

package com.example.search.constant;

public class EsConstant {
    public static final String PRODUCT_INDEX="product";//sku数据在es中的索引
}

package com.example.search.config;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client. RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class GulimallElasticSearchConfig {
    public static final RequestOptions COMMON_OPTIONS;
    static {
        RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
        COMMON_OPTIONS = builder.build();
    }

    @Bean
    public RestHighLevelClient esRestClient(){
        RestClientBuilder builder=null;
        //final String hostname, final int port, final String scheme
        builder = RestClient.builder(new HttpHost("192.168.56.10", 9200, "http"));
        RestHighLevelClient client = new RestHighLevelClient(builder);
        return client;
    }
}

public class ProductSaveServiceIm

@Slf4j
@Service
public class ProductSaveServiceImpl implements ProductSaveService {
   @Autowired
    RestHighLevelClient restHighLevelClient;

    @Override
    public boolean productStatusUp(List<SkuEsModel> skuEsModes) throws IOException {
        //保存到es
        //1.给es中建立索引。product，建立好映射关系
        BulkRequest bulkRequest = new BulkRequest();
        for(SkuEsModel model:skuEsModes){
            //构造保存请求
            IndexRequest indexRequest = new IndexRequest(EsConstant.PRODUCT_INDEX);
            indexRequest.id(model.getSkuId().toString());
            String s = JSON.toJSONString(model);
            indexRequest.source(s, XContentType.JSON);

            bulkRequest.add(indexRequest);//
        }
       //2.给es中保存这些数据
        //批量操作sku数据
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS);
       //统计上架错误的sku
        //TODO  如果批量错误
        boolean b = bulk.hasFailures();
        List<String> collect = Arrays.stream(bulk.getItems()).map(item -> {
            return item.getId();

        }).collect(Collectors.toList());
        log.info("商品上架完成: {},返回数据:{}",collect,bulk.toString());
        return b;
    }

}

SpuInfoServiceImpl

 @Override
    public void up(Long spuId) {

         //1.查出当前spuid对应的所有sku信息，品牌的名字
        List<SkuInfoEntity>skus =  skuInfoService.getSkusBySpuId(spuId);

        //TODO 4.查出当前sku所有可以被用来检索的规格属性信息pms_product_attr_value
        //获取当前spu对应的的所有attr信息
        List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrListForSpu(spuId);
       //挑出所有检索的信息
         List<Long> attrIds = baseAttrs.stream().map((attr -> {
            return attr.getAttrId();
        })).collect(Collectors.toList());
        //pms_attr,在指定的所有属性集合里边，找出检索属性idSet
        List<Long> searchAttrIds =  attrService.selectSearchAttrIds(attrIds);
       Set<Long> idSet =  new HashSet<>(searchAttrIds);
//         Set<Long> idSet = searchAttrIds.stream().collect(Collectors.toSet());
        List<SkuEsModel.Attrs> attrsList =  baseAttrs.stream().filter(item -> {
            return idSet.contains(item.getAttrId());
        }).map(item -> {
            SkuEsModel.Attrs attrs1 = new SkuEsModel.Attrs();
            BeanUtils.copyProperties(item, attrs1);
            return attrs1;
        }).collect(Collectors.toList());
        //拿到所有sku的id
        List<Long> skuIdList = skus.stream().map(SkuInfoEntity::getSkuId).collect(Collectors.toList());
        //TODO 1。发送远程调用，库存系统查询是否有库存
        Map<Long, Boolean> stockMap =null;
        try{
            R r = wareFeignService.getSkusHasStock(skuIdList);
//            TypeReference<List<SkuHasStockVo>> typeReference = new TypeReference<List<SkuHasStockVo>>() {};
//            stockMap = skusHasStock.getData(typeReference).stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));

         //SkuHasStockVo::getSkuId --->key
          TypeReference<List<SkuHasStockVo>>  typeReference = new TypeReference<List<SkuHasStockVo>>(){};
            stockMap = r.getData(typeReference).stream().collect(Collectors
                    .toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock()));
        }catch(Exception e){
            log.error("库存查询异常：原因{}",e);
        }



        //2.封装每个sku的信息
        Map<Long,Boolean> finalStockMap = stockMap;
        List<SkuEsModel>upProducts= skus.stream().map(sku -> {
            //组装需要的数据pms_sku_info
            SkuEsModel esModel = new SkuEsModel();
            BeanUtils.copyProperties(sku,esModel);
            esModel.setSkuPrice(sku.getPrice());
            esModel.setSkuImg(sku.getSkuDefaultImg());
           //hasStokc,hotSore 设置库存信息
            if(finalStockMap ==null){
                esModel.setHasStock(true);
            }else{
                esModel.setHasStock(finalStockMap.get(sku.getSkuId()));
            }

            //TODO 2.热度评分 刚上架是0
            esModel.setHotScore(0L);
            //TODO  3.查询品牌和分类的名字信息
           BrandEntity brand = brandService.getById(esModel.getBrandId());
           esModel.setBrandName(brand.getName());
           //esModel.setBrandId(brand.getBrandId());//
           esModel.setBrandImg(brand.getLogo());

            //分类信息
            CategoryEntity category = categoryService.getById(esModel.getCatalogId());
           // esModel.setCatalogId(category.getCatId());//
            esModel.setCatalogName(category.getName());
          //设置检索属性
           esModel.setAttrs(attrsList);
           // BeanUtils.copyProperties(sku,esModel);//zheli ?
            return esModel;
        }).collect(Collectors.toList());
         //TODO 5.将数据发送给es进行保存：gulimall-search
       R r =  searchFeignSerice.productStatusUp(upProducts);
       if(r.getCode()==0) {
           //远程调用成功
           //TODO 6.修改当前spu的状态 pms_spu_info
          baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode());
       }else{
           //远程调用失败
           // TODO 重复调用？；接口幂等性 重试机制
       }
    }

用于数据逆转--添加getData和setData

public class R extends HashMap<String, Object> {
	private static final long serialVersionUID = 1L;
//利用fastJSON进行逆转
	public<T> T getData(TypeReference<T> typeReference){
		Object data= get("data");//默认是map类型
		 String s = JSON.toJSONString(data);
		T t = JSON.parseObject("s",typeReference);
		return t;

	}
public R setData(Object data){
	put("data",data);
	return this;
}