ES基本使用

ES基本使用

一、ES环境搭建

1.1 导入依赖

<dependencies>
        <!--springboot依赖-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <!--ES依赖包-->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
        </dependency>
        <!--nacos服务注册发现依赖-->
        <dependency>
            <groupId>com.alibaba.cloud</groupId>
            <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
        </dependency>
        <!--feign远程调用-->
        <dependency>
            <groupId>com.hmall</groupId>
            <artifactId>feign-api</artifactId>
            <version>1.0</version>
        </dependency>
        <!--fastjson-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.79</version>
        </dependency>
        <!--commons工具包-->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
        </dependency>
    </dependencies>

1.2 配置文件

server:
  port: 8084 #es服务的公共端口
spring:
  application:
    name: searchservice #es工程服务名称
  cloud:
    nacos:
      server-addr: localhost:8848 #nacos服务端地址

logging: #日志
  level:
    com.hmall: debug
  pattern:
    dateformat: HH:mm:ss:SSS

1.3 ES对象初始化

指定es服务的地址，可以将bean写在启动类下

@SpringBootApplication
@EnableFeignClients("com.hmall.common.item.feign")
public class SearchApplication {
    public static void main(String[] args) {
        SpringApplication.run(SearchApplication.class,args);
    }

    @Bean
    public RestHighLevelClient restClient(){
        return new RestHighLevelClient(RestClient.builder(HttpHost.create("http://192.168.43.31:9200")));
    }

}

二、基本使用

2.1 es数据同步

2.1.1 批量导入

2.1.1.2 非定时导入

//此处使用openfeign远程调用了itemservice的list方法，该方法会分页查询mysql中的数据以及总数据数量
@Autowired
private ItemFeign itemFeign;

@Override
public void batchImportItem() throws IOException {
    //先默认读取第一页获取总数据条数
    int page = 1, size = 1000, pages;

    do {

        //1.根据总数据条数 计算 总页数
        PageDTO<?> dto = itemFeign.list(page, size);
        //总条数
        Long total = dto.getTotal();
        //总页数
        pages = (int) (total % size == 0 ? total / size : total / size + 1);

        //2.将本次查询的数据导入到es中
        //2.1 将dto -》 itemDOC
        List<ItemDoc> itemDocs = JSON.parseArray(JSON.toJSONString(dto.getList()), ItemDoc.class);
        //2.2 批量导入到es
        batchImport(itemDocs);

        //3.1.判断本次页是否超出最大页
        page++;

        //3.2.超出则跳出循环
    } while (page <= pages);
}


	/**
     * 批量导入到es
     *
     * @param docs
     */
private void batchImport(List<ItemDoc> docs) throws IOException {

    //创建bulkRequest对象 批量导入对象
    BulkRequest bulkRequest = new BulkRequest();

    //将所有导入对象添加到bulkRequest对象中
    for (ItemDoc doc : docs) {
        //处理单词联想属性
        doc.setSuggestion();

        IndexRequest request = new IndexRequest("shop").id(doc.getId());
        request.source(JSON.toJSONString(doc), XContentType.JSON);
        bulkRequest.add(request);
    }

    restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}

@Data
@AllArgsConstructor
@NoArgsConstructor
public class ItemDoc {
    private String id;
    private String name;
    private Long price;
    private String image;
    private String category;
    private String brand;
    private Integer sold;
    private Integer commentCount;
    private Boolean isAD;
    private List<String> suggestion;

    /**
     * 单词联想的数据处理 = name + 分类 + 品牌
     */
    public void setSuggestion(){
        suggestion = new ArrayList<>();
        suggestion.add(name);
        suggestion.add(category);
        suggestion.add(brand);
    }

}

2.1.1.2 基于xxl-job定时批量同步

第一种非定时导入不太适合真实业务场景，一般都是在某个时间点定时批量导入数据到ES

只不过需要在xxl-job中创建一个定时任务，指定什么时间执行，执行什么任务

/**
 * 定时任务到达时间后执行
 *
 * @author lmz
 */
@Component
@Slf4j
public class SyncIndexTask {

    /**
     * feign远程调用
     */
    @Autowired
    private IArticleClient articleClient;

    /**
     * es
     */
    @Autowired
    private RestHighLevelClient restHighLevelClient;

    /**
     * 线程池
     */
    private static ThreadPoolExecutor pool = new ThreadPoolExecutor(
            5,
            10,
            0L,
            TimeUnit.SECONDS,
            new ArrayBlockingQueue<>(5),
            new ThreadPoolExecutor.AbortPolicy()
    );


    /***
     * 同步索引任务
     *  1)当数量大于100条的时候，才做分片导入，否则只让第1个导入即可
     *      A:查询所有数据量 ->searchTotal total>100 [判断当前分片不是第1个分片]
     *      第N个分片执行数据处理范围-要计算   确定当前分片处理的数据范围  limit #{index},#{size}
     *                                                                [index-范围]
     *
     *      B:执行分页查询-需要根据index判断是否超过界限，如果没有超过界限，则并开启多线程，分页查询，将当前分页数据批量导入到ES
     *
     *      C:在xxl-job中配置作业-策略：分片策略
     *
     */
    @XxlJob("syncIndex")
    public void syncIndex()  {
        //1、获取任务传入的参数   {"minSize":100,"size":10}
        String jobParam = XxlJobHelper.getJobParam();
        Map<String,Integer> jobData = JSON.parseObject(jobParam,Map.class);
        //分片处理的最小总数据条数
        int minSize = jobData.get("minSize");
        //分页查询的每页条数   小分页
        int size =  jobData.get("size");

        //2、查询需要处理的总数据量  total=IArticleClient.searchTotal()
        Long total = articleClient.total();

        //3、判断当前分片是否属于第1片，不属于，则需要判断总数量是否大于指定的数据量[minSize]，大于，则执行任务处理，小于或等于，则直接结束任务
        //当前节点的下标
        int cn = XxlJobHelper.getShardIndex();
        if(total<=minSize && cn!=0){
            //结束
            return;
        }
        //4、执行任务   [index-范围]   大的分片分页处理
        //4.1：节点个数
        int n = XxlJobHelper.getShardTotal();
        //4.2：当前节点处理的数据量
        int count = (int) (total % n==0? total/n :  (total/n)+1);
        //4.3：确定当前节点处理的数据范围
        //从下标为index的数据开始处理  limit #{index},#{count}
        int indexStart = cn*count;
        //最大的范围的最后一个数据的下标
        int indexEnd = cn*count+count-1;
        //5.小的分页查询和批量处理
        //第1页的index
        int index =indexStart;

        log.info("分片个数是【{}】,当前分片下标【{}】，处理的数据下标范围【{}-{}】",n,cn,indexStart,indexEnd);
        do {
            //=============================================小分页================================
            //5.1:分页查询
            //5.2:将数据导入ES
            push(index,size,indexEnd);

            //5.3:是否要查询下一页 index+size
            index = index+size;
        }while (index<=indexEnd);
    }


    /**
     * 数据批量导入
     * @param index
     * @param size
     * @param indexEnd
     * @throws IOException
     */
    public void push(int index,int size,int indexEnd)  {

        pool.execute(()->{
            log.info("当前线程处理的分页数据是【index={},size={}】",index,(index+size>indexEnd? indexEnd-index+1 : size));
            //1)查询数据库数据
            List<SearchArticleVo> searchArticleVos = articleClient.pageSearch(index, index+size>indexEnd? indexEnd-index+1 : size);  //size可能越界
            // 第1页  index=0
            //       indexEnd=6
            // 第2页  index=5
            //       indexEnd-index+=2
            //2)创建BulkRequest - 刷新策略
            BulkRequest bulkRequest = new BulkRequest()
                    //刷新策略-立即刷新
                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
            for (SearchArticleVo searchArticleVo : searchArticleVos) {
                //A:创建XxxRequest
                IndexRequest indexRequest = new IndexRequest("hmtt")
                        //B:向XxxRequest封装DSL语句数据
                        .id(searchArticleVo.getId().toString())
                        .source(com.alibaba.fastjson.JSON.toJSONString(searchArticleVo), XContentType.JSON);

                //3)将XxxRequest添加到BulkRequest
                bulkRequest.add(indexRequest);
            }

            //4)使用RestHighLevelClient将BulkRequest添加到索引库
            if(searchArticleVos!=null && searchArticleVos.size()>0){
                try {
                    restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

2.1.2 增量导入

增量导入可以基于 rabbitmq 来做异步通信，当数据库中数据更改时就将更新的数据发送到rabbitmq中，es微服务监听rabbitmq然后进行消费，判断以下此时是更新还是删除还是添加，或者监听不同队列也是可以的

1) 后台增删改

    @PostMapping
    public void saveHotel(@RequestBody Hotel hotel){
        hotel.setId((long) (Math.random() * 10000));
        hotelService.save(hotel);

        //发送消息到mq指定队列中
        rabbitTemplate.convertAndSend(MQConstants.HOTEL_EXCHANGE,MQConstants.HOTEL_ADD_KEY, JSON.toJSONString(hotel));
    }

    @PutMapping()
    public void updateById(@RequestBody Hotel hotel){
        if (hotel.getId() == null) {
            throw new InvalidParameterException("id不能为空");
        }
        hotelService.updateById(hotel);

        //发送消息到mq指定队列中
        rabbitTemplate.convertAndSend(MQConstants.HOTEL_EXCHANGE,MQConstants.HOTEL_UPDATE_KEY, JSON.toJSONString(hotel));

    }

    @DeleteMapping("/{id}")
    public void deleteById(@PathVariable("id") Long id) {
        hotelService.removeById(id);

        //发送消息到mq指定队列中
        rabbitTemplate.convertAndSend(MQConstants.HOTEL_EXCHANGE,MQConstants.HOTEL_DELETE_KEY, JSON.toJSONString(id));

    }

2) es同步

2.1) 监听器

@Component
public class HotelMqListen {

    @Autowired
    private IHotelService service;

    //监听添加酒店
    @RabbitListener(bindings = @QueueBinding(//绑定队列 和 交换机
            value = @Queue(MQConstants.HOTEL_ADD_QUERY),//定义添加时队列
            exchange = @Exchange(value = MQConstants.HOTEL_EXCHANGE,type = ExchangeTypes.DIRECT),//定义添加时交换机
            key = {MQConstants.HOTEL_ADD_KEY}//定义添加路由标识
    ))
    public void hotelAdd(String msg) throws IOException {
        //解析消息为 HotelDoc对象
        Hotel hotel = JSON.parseObject(msg, Hotel.class);
        HotelDoc doc = new HotelDoc(hotel);
        //添加
        service.add(doc);
    }

    //监听修改酒店
    @RabbitListener(bindings = @QueueBinding(//绑定队列 和 交换机
            value = @Queue(MQConstants.HOTEL_UPDATE_QUERY),
            exchange = @Exchange(MQConstants.HOTEL_EXCHANGE),
            key = {MQConstants.HOTEL_UPDATE_KEY}
    ))
    public void hoteUpdate(String msg) throws IOException {
        //解析消息为 HotelDoc对象
        Hotel hotel = JSON.parseObject(msg, Hotel.class);
        HotelDoc doc = new HotelDoc(hotel);
        //添加
        service.updateHotel(doc);
    }

    //监听删除酒店
    @RabbitListener(bindings = @QueueBinding(//绑定队列 和 交换机
            value = @Queue(MQConstants.HOTEL_DELETE_QUERY),
            exchange = @Exchange(value = MQConstants.HOTEL_EXCHANGE),
            key = {MQConstants.HOTEL_DELETE_KEY}
    ))
    public void hotelDelete(String id) throws IOException {
        service.deleteHotel(id);
    }

}

2.2) 服务端

    //同步数据库修改操作
    @Override
    public void updateHotel(HotelDoc doc) throws IOException {
        //封装XXXRequest对象 指定索引库 & id
        UpdateRequest request = new UpdateRequest("hotel",doc.getId().toString());
        //XXXRequest对象封装DSL语句
        request.doc(JSON.toJSONString(doc),XContentType.JSON);
        //发送http请求，执行es操作
        client.update(request,RequestOptions.DEFAULT);
    }

    //同步数据库删除操作
    @Override
    public void deleteHotel(String id) throws IOException {

        DeleteRequest request = new DeleteRequest("hotel").id(id);

        client.delete(request,RequestOptions.DEFAULT);

    }

    //同步数据库添加操作
    @Override
    public void add(HotelDoc doc) throws IOException {
        //IndexRequest
        IndexRequest request = new IndexRequest("hotel").id(doc.getId().toString());

        //封装dsl语句
        request.source(JSON.toJSONString(doc), XContentType.JSON);

        //发送请求
        client.index(request,RequestOptions.DEFAULT);
    }

2.3) 常量类

//MQ常量类
public class MQConstants {

    public final static String HOTEL_EXCHANGE = "hotel.router";
    
    public final static String HOTEL_ADD_QUERY = "hotel.add.query";
    public final static String HOTEL_UPDATE_QUERY = "hotel.update.query";
    public final static String HOTEL_DELETE_QUERY = "hotel.delete.query";

    public final static String HOTEL_ADD_KEY = "hotel.add";
    public final static String HOTEL_UPDATE_KEY = "hotel.update";
    public final static String HOTEL_DELETE_KEY = "hotel.delete";

}

2.2 自动补全功能

搜索框用户输入时会自动单词联想，让用户查询

@Override
public List<String> suggestion(String key) throws IOException {
    //SearchRequest对象	指定索引库
    SearchRequest request = new SearchRequest("shop");

    //自动补全查询DSL语句封装
    request.source().suggest(new SuggestBuilder().addSuggestion(
        //查询后的列名称
        "suggestion",
        //查询字段
        SuggestBuilders.completionSuggestion("suggestion")
        //查询的key
        .prefix(key)
        //跳过重复
        .skipDuplicates(true)
        //获取前10条，联想几条数据
        .size(10)
    ));

    //发送请求，结果集解析
    SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);

    //此处需要获取前面设置的查询后的列名称
    CompletionSuggestion titleSuggest = response.getSuggest().getSuggestion("suggestion");
    return titleSuggest.getOptions().stream()
        .map(text -> text.getText().toString())
        .collect(Collectors.toList());
}

2.3 基本搜索功能

比如用户输入关键字搜索、按照指定类搜索、指定价格排序、高亮数据显示等等

public PageDTO<ItemDoc> list(RequestParams params) throws IOException {
        //封装Request对象 指定索引库
        SearchRequest request = new SearchRequest("shop");

        //Request对象封装DSL语句 match
        basicBuild(params,request);

        //设置分页
        //起始页 （page - 1）*size
        int page = (params.getPage() - 1) * params.getSize();
        request.source().from(page).size(params.getSize());

        //发送http请求 执行es操作，获取结果
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);

        //解析结果返回
        return parseResult(response,params);
    }

    /**
     * 搜索条件构造
     * @param params
     * @param request
     */
    private void basicBuild(RequestParams params,SearchRequest request){

        //复合查询
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();

        //判断用户是否输入查询关键字
        if (ObjectUtils.isEmpty(params.getKey())){
            //未输入就查询所有
            boolQuery.must(QueryBuilders.matchAllQuery());
        }else{
            //输入则根据key查询
            boolQuery.must(QueryBuilders.matchQuery("name",params.getKey()));
        }

        //品牌过滤
        if (ObjectUtils.isNotEmpty(params.getBrand())){
            boolQuery.filter(QueryBuilders.termQuery("brand",params.getBrand()));
        }

        //分类过滤
        if (ObjectUtils.isNotEmpty(params.getCategory())){
            boolQuery.filter(QueryBuilders.termQuery("category",params.getCategory()));
        }

        //价格过滤
        if (ObjectUtils.allNotNull(params.getMinPrice(),params.getMaxPrice())){
            //将前端传的价格单位元 -》 单位分
            params.setMinPrice(params.getMinPrice() * 100);
            params.setMaxPrice(params.getMaxPrice() * 100);
            boolQuery.filter(QueryBuilders.rangeQuery("price")
                    .gt(params.getMinPrice())
                    .lte(params.getMaxPrice())
            );
        }

        //销量降序 || 价格升序
        if (StringUtils.equals(params.getSortBy(),"sold")){
            request.source().sort("sold",SortOrder.DESC);
        }

        if (StringUtils.equals(params.getSortBy(),"price")){
            request.source().sort("price",SortOrder.ASC);
        }

        //开启高亮
        request.source().highlighter(
                new HighlightBuilder().field("name")
                        .preTags("<span style=\"color:red;font-size:18px;\">")
                        .postTags("</span>")
        );

        //相关性加分
        FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(
                //原始查询条件
                boolQuery,
                new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
                        new FunctionScoreQueryBuilder.FilterFunctionBuilder(
                                //筛选条件
                                QueryBuilders.termQuery("isAD",true),
                                //权重加分
                                ScoreFunctionBuilders.weightFactorFunction(10)
                        )
                }
                //运算方式
        ).boostMode(CombineFunction.SUM);

        request.source().query(functionScoreQueryBuilder);
    }

    /**
     * 解析分页查询结果
     * @param response
     * @param params 当需要处理距离排序时会使用到
     * @return
     */
    private PageDTO<ItemDoc> parseResult(SearchResponse response,RequestParams params){
        //获取hits
        SearchHits hits = response.getHits();
        //总记录数
        long total = hits.getTotalHits().value;

        //获取hits.hits集合数据
        ArrayList<ItemDoc> docs = new ArrayList<>();
        for (SearchHit hit : hits.getHits()) {
            //获取每个对象的source
            String json = hit.getSourceAsString();
            ItemDoc doc = JSON.parseObject(json, ItemDoc.class);

            //获取高亮数据，判断是否有高亮数据，有就获取
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            if(ObjectUtils.allNotNull(highlightFields, highlightFields.get("name"))){
                String highName = StringUtils.join(
                        highlightFields.get("name").getFragments(), "....");
                //赋值给HotelDoc对象
                doc.setName(highName);
            }

            docs.add(doc);
        }

        return new PageDTO<ItemDoc>(total,docs);
    }

@Data
@NoArgsConstructor
@AllArgsConstructor
public class RequestParams {

    private String key;
    private Integer page;
    private Integer size;
    private String sortBy;
    private String category;
    private String brand;
    private Long minPrice;
    private Long maxPrice;

}

2.4 过滤项聚合功能

根据es数据动态显示过滤项，es聚合查询过滤项

public Map<String, List<String>> filters(RequestParams params) throws IOException {
        //创建XXXRequest对象
        SearchRequest request = new SearchRequest("shop");
        //封装DSL语句
        //1.基本查询
        basicBuild(params,request);

        //2.聚合查询
        //2.0 不返回查询的结果，只要聚合结果
        request.source().size(0);

        SearchSourceBuilder source = request.source();
        //2.2 品牌查询
        aggBuild("brandAgg","brand",source);

        //2.3 分类查询
        aggBuild("categoryAgg","category",source);

        //发送http请求查询
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);

        //解析结果集然后返回
        Map<String,List<String>> map = new HashMap<>();
        map.put("category",parseFilterResult(response,"categoryAgg"));
        map.put("brand",parseFilterResult(response,"brandAgg"));
        return map;
    }

    /**
     * 解析filter结果集
     * @param response
     * @param aggName 聚合名称
     * @return
     */
    private List<String> parseFilterResult(SearchResponse response, String aggName){
        ParsedStringTerms agg = response.getAggregations().get(aggName);
        return agg.getBuckets().stream()
                .map(k -> k.getKeyAsString())
                .collect(Collectors.toList());
    }

    /**
     * 聚合查询构造
     * @param aggName 聚合名称
     * @param field 聚合字段
     * @param source
     */
    private void aggBuild(String aggName,String field,SearchSourceBuilder source){
        source.aggregation(AggregationBuilders
                .terms(aggName)
                .field(field)
                .size(20)//显示几条
                .order(BucketOrder.count(false)));
    }