ES自己手动高亮

最新推荐文章于 2024-08-12 21:17:32 发布

你看我圆嘛

最新推荐文章于 2024-08-12 21:17:32 发布

阅读量1.5k

点赞数

分类专栏： java es 文章标签： elasticsearch java 数据库

本文链接：https://blog.csdn.net/weixin_47669567/article/details/127300752

版权

java 同时被 2 个专栏收录

5 篇文章 0 订阅

订阅专栏

3 篇文章 0 订阅

订阅专栏

背景: es的高亮真的是一言难尽,经常出现各种各样的高亮异常. 如高亮错位高亮词错误等等.而且用wildcardQuery 等也无法高亮. 可能是我技术不精吧,总是调不好这玩意,因此决定手写高亮 .

废话不多说直接上代码:
1:第一步处理高亮词:
这个跟各位的业务有关系,比如有没有用到近义词有没有什么繁简纠错等等的转换. 这一步,就是要把输入词keyword 转成需要高亮的词数组[以下代码是我自己业务的处理流程, 各位将就着看,自己理解,对着自己的业务,将就着改]:

    private void checkHighlight (EsPageDTO esPageDTO,IndexQueryDTO objDTO,EsSearchDTO esSearchDTO) {
        if(esPageDTO.getResponseData()== null){
            return;
        }
       Set<String> words = new HashSet<>();
        //查询关键词
        String word = objDTO.getKeyword();
        //实际词
        String actualWord = esPageDTO.getActualWord();
        //标记 查询词
        String keyword = StringUtils.isBlank(actualWord) ? word: actualWord;

        //高亮配置
        QueryHighlightConfigDTO highlightconfig = esSearchDTO.getHighlightconfig();
        //判断是否是高级检索
        Map<String, Object> seniorParams = objDTO.getSeniorParams();
        if(seniorParams == null){
            //如果是精准检索
            if(objDTO.getExactQuery()){
                words.add(word);
            }else{
                //繁体字
                List<String> tWords = new ArrayList<>();
                //添加查询词
                words.add(keyword);
                //从缓存中获取查询词 拆词(选 word 还是 actualWord )
                if(mapCache.get(EsConstants.ES_CACHE_WORD_CUT+keyword) != null){
                    List<String> cutWords = new ArrayList<>();
                    List<OneToManyTrunkDTO>  integration = ((List<OneToManyTrunkDTO>) mapCache.get(EsConstants.ES_CACHE_WORD_CUT + keyword));
                    integration.forEach(item->{
                        String s = item.getWord();
                        if(item.getSeal().contains(ChineseS2THandle.SEAL_T)){
                            tWords.add(s);
                        }else{
                            if(s.contains(",")){
                                cutWords.addAll(Arrays.asList(s.split(",")));
                            }else{
                                cutWords.add(s);
                            }
                        }
                    });
                    words.addAll(cutWords);
                }
                //让这些words经受ES smart分词器洗礼 [繁体字可免除洗礼,因为ik分词器压根无法对繁体字进行准确分词]
                words.addAll(esPlusUtils.testAnalyzer(words.toArray(new String[0]), Analyzer.ik_smart.name())) ;
                //重新添加 tWord
                if(CollectionUtils.isNotEmpty(tWords)){
                    words.addAll(tWords);
                }

                //用新容器 是为了防止 ConcurrentModificationException 因为是异步 words 可能会一边在add 一边在read
                List<String> forFull2Half = new ArrayList<>(words);
                List<String> forHalf2Full = new ArrayList<>(words);

                //全角半角转换[转半角]
                CompletableFuture<Set<String>> full2HalfFuture = CompletableFuture.supplyAsync(() ->
                        this.getHalfWord(forFull2Half)).whenComplete((v, t) ->{
                    words.addAll(v);
                });
                //全角半角转换[转全角]
                CompletableFuture<Set<String>> half2FullFuture = CompletableFuture.supplyAsync(() ->
                        this.getFullWord(forHalf2Full)).whenComplete((v, t) ->{
                    words.addAll(v);
                });
                CompletableFuture.allOf(full2HalfFuture,half2FullFuture).join();
            }
        }
        List<SearchResultVO> responseData = esPageDTO.getResponseData();
        for (SearchResultVO item : responseData) {
            Class<SearchResultVO> itemClass = SearchResultVO.class;
            //获取高亮字段
            List<String> mappingFields = highlightconfig.getMappingFieldsAll();
            Map<String, String> mappingFieldNameAndTypeAll = highlightconfig.getMappingFieldNameAndTypeAll();
            //默认都会给的 起码有个titlt 和 content
            for (String highlightFiled : mappingFields) {
                try {
                    Set<String> queryWords;
                    Field field = itemClass.getDeclaredField(highlightFiled);
                    field.setAccessible(true);
                    String text = field.get(item).toString();
                    //获取 查询词
                    if(seniorParams == null){
                        queryWords = words;
                    }else{
                        Object wordObj = seniorParams.get(highlightFiled);
                        if(null == wordObj || StringUtils.isBlank(wordObj.toString())){
                            //如果查询词是空 就不考虑了
                            continue;
                        }
                        queryWords = new HashSet<>(Collections.singletonList(wordObj.toString()));
                        //这里 没有 搞花里胡哨 的 自定义词拆分 但是  还是会被 分词 所以 还是要走一下 ik
                        queryWords = new HashSet<>(esPlusUtils.testAnalyzer(queryWords.toArray(new String[0]), Analyzer.ik_smart.name()));
                    }
                    String lastText = this.buildHighlight(highlightconfig, text, queryWords,EsFiledsEnum.TEXT.getType().equals(mappingFieldNameAndTypeAll.get(highlightFiled)));
                    field.set(item,lastText);
                } catch (NoSuchFieldException | IllegalAccessException ignored) {
                }
            }
        }
    }

第二步高亮

private String buildHighlight(QueryHighlightConfigDTO highlightconfig,String text,Set<String> wordsparam,Boolean cut){
        //高亮前缀
        String preTags = String.format("<span style=\"color:%s\" >",highlightconfig.getColour()) ;
        //高亮后缀
        String postTags = "</span>";

       // 已经高亮的 就不处理了  -- 目前已经放弃 ES 高亮了
        if(text.contains("<span")){
            //应该校准一下 但是太复杂了 以后再说吧 尽量在前面避免 高亮不准 毕竟高亮不准 肯定是有异常原因的
            return text;
        }
        //把words 进行排序 长字段 优先
        List<String> words = wordsparam.stream().sorted(Comparator.comparing(String::length).reversed()).collect(Collectors.toList());
        if(!cut){
            for (String s : words) {
                if(text.contains(s)){
                    text = text.replaceAll(s,preTags+s+postTags);
                }
            }
            return text;
        }
        //片段大小
        int fragmentSize = highlightconfig.getFragmentSize() == null ? 50 : highlightconfig.getFragmentSize();
        //关键词字数
        int numOfFragment = highlightconfig.getNumOfFragment() == null ? 3:highlightconfig.getNumOfFragment();
        //未匹配返回长度
        int noMatchSize = highlightconfig.getNoMatchSize() == null ? 0 : highlightconfig.getNoMatchSize() ;

        List<Integer[]> partSection = new ArrayList<>();
//        List<String> maxWords = new ArrayList<>();
//        words.forEach(item->maxWords.addAll(Arrays.asList(item.split(""))));
        int alreadyFindnumOfFragment = 0;
        //已使用的Words
        List<String> alreadyUseWord = new ArrayList<>();
        for (String s : words) {
            if(text.contains(s)){
                //根据关键词字数 去截取片段
                for (int i = 0; i <numOfFragment;i++) {
                    if(alreadyFindnumOfFragment == numOfFragment){
                        break;
                    }
                    int index = find(text, s, i + 1);
                    if(index == -1){
                        break;
                    }
                    int section = index+s.length();
                    //片段大小 前后 一半 ?
                    //计算 fragmentSize 的 前一半 和 后一半
                    int firstHalf  = fragmentSize/2;
                    int secondHalf  = fragmentSize/2 + fragmentSize%2;
                    int start = Math.max(section - firstHalf, 0);
                    int end = Math.min(section + secondHalf, text.length()-1);
                    partSection.add(new Integer[]{start,end});
                    alreadyUseWord.add(s);
                    alreadyFindnumOfFragment++;
                }
            }else{
                // sb.append(text.substring(Math.min((noMatchSize + 1), text.length()+1)));
            }
        }
        //整合区间
        List<Integer[]> lastSection = new ArrayList<>();
        if(CollectionUtils.isEmpty(partSection)){
            return  text;
        }
        //按区间起始时间给区间排序
        partSection.sort(Comparator.comparing(item->item[0]));
        Integer lastStart  = 0;
        Integer lastEnd = 0;
        for (int i = 0; i < partSection.size(); i++) {
            Integer currentStart = (partSection.get(i))[0];
            Integer currentEnd = (partSection.get(i))[1];
            if(i == 0){
                lastStart = currentStart;
                lastEnd =  currentEnd;
            }else{
                //有交集(不可能出现等于,因为字符都是有长度的)
                if(lastEnd > currentStart){
                    lastEnd = currentEnd;
                }else{
                    lastSection.add(new Integer[]{lastStart,lastEnd});
                    lastStart = currentStart;
                    lastEnd = currentEnd;
                }
            }
            //最后了
            if(i ==  partSection.size()-1){
                lastSection.add(new Integer[]{lastStart,lastEnd});
            }
        }
        StringBuilder sb = new StringBuilder();
        for (Integer[] integers : lastSection) {
            sb.append(text, integers[0], integers[1]+1);
        }
        String result = sb.toString();
        for (String s : alreadyUseWord) {
            try {
                result = result.replace(s,preTags+s+postTags);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
       return result;
    }

 /**
     * 字符串中某字符 第N次出现的位置
     * */
    public static int find( String text, String part, int n) {
        int x = text.indexOf(part);;
        if (n == 1) {
            return x;
        }
        for (int i = 1; i < n; i += 1) {
            x = text.indexOf(part, x + 1);
        }
        return x;
    }