[搜索引擎] Elasticsearch搜索引擎 spring-boot orm工具

项目目录

包括ElasticSearch连接 参数映射 结果集映射 关键词高亮

一、Elasticsearch配置类

使用HighLevelRestClient作为Elasitcsearch的连接

@Configuration
public class HighLevelRestClientConfig {

    private final static Logger log = LoggerFactory.getLogger(HighLevelRestClientConfig.class);

    @Value("${elasticsearch.cluster-nodes}")
    private String[] host;

    private volatile RestHighLevelClient restHighLevelClient = null;

    @ConditionalOnMissingBean
    @Bean
    public RestHighLevelClient getClient() {
        if (Objects.nonNull(restHighLevelClient)) {
            return restHighLevelClient;
        }
        if (Objects.isNull(restHighLevelClient)) {
            try {
                restHighLevelClient = creatClient();
            } catch (Exception e) {
                log.error(ErrorUtil.getErrorStack(e, "初始化elasticsearch client 失败!"));
            }
        }
        return restHighLevelClient;
    }

    private RestHighLevelClient creatClient() {
        List<HttpHost> httpHostList = new ArrayList<>();
        Arrays.stream(host).forEach(v -> {
            if (StringUtils.contains(v, ":")) {
                log.info("elasticsearch connect host: " + v);

                HttpHost httpHost = new HttpHost(
                        StringUtils.split(v, ":")[0],
                        Integer.parseInt(StringUtils.split(v, ":")[1]),
                        "http");
                httpHostList.add(httpHost);
            }
        });

        RestClientBuilder restClient = null;
        if (httpHostList.size() == 1) {
            restClient = RestClient.builder(httpHostList.get(0));
        } else {
            restClient = RestClient.builder(httpHostList.toArray(new HttpHost[]{}));
        }

        Objects.requireNonNull(restClient);

        // 定义监听器，节点出现故障会收到通知。
        restClient.setFailureListener(new RestClient.FailureListener() {
            @Override
            public void onFailure(Node node) {
                super.onFailure(node);
                HttpHost host = node.getHost();
                String name = node.getName();
                log.error("elasticsearch节点出现故障, host: " + host + "  name: " + name);
            }
        });

        // 定义节点选择器 这个是跳过data=false，ingest为false的节点
        restClient.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
        // 定义默认请求配置回调
        restClient.setRequestConfigCallback(requestConfigBuilder -> {
            return requestConfigBuilder.setConnectTimeout(90000) // 连接超时（默认为1秒）
                    .setSocketTimeout(30000); // 套接字超时（默认为30秒）
        });

        RestHighLevelClient client = new RestHighLevelClient(restClient);
        return client;
    }

二、核心接口

主要方法

init()
初始化index,type信息

Page<T> searchPage(SearchSourceBuilder builder, Pageable pageable)
使用spring-data-common 分页

Optional<T> searchOne(SearchSourceBuilder builder)
搜索返回单个结果 Optional包装

long count(SearchSourceBuilder searchSourceBuilder)
查询结果数量

String save(T entity)
插入和更新数据 返回id

void updateByQuery(QueryBuilder queryBuilder, T entity)
批量更新

@NoRepositoryBean
public interface ElasticsearchCrudRepository<T> extends Repository<T, String> {

    /**
     * TODO 初始化方法要设置index和type 必须实现
     *
     * PostConstruct 
     * public void init() {
     * this.setIndexName("odata_unstructured_document");
     * this.setTypeName("doc");
     * }
     */
    void init();

    /**
     * 根据id查询数据
     *
     * @param entity
     * @return
     */
    String getId(T entity);

    /**
     * 分页搜索
     *
     * @param builder
     * @param pageable
     * @return
     */
    Page<T> searchPage(SearchSourceBuilder builder, Pageable pageable);

    /**
     * 搜索一个
     *
     * @param builder
     * @return
     */
    Optional<T> searchOne(SearchSourceBuilder builder);

    /**
     * 获取词向量数据
     *
     * @param termVectorsRequest
     * @return
     * @throws IOException
     */
    TermVectorsResponse getTermVector(TermVectorsRequest termVectorsRequest) throws IOException;

    Map<String, List<TermVectorsResponse.TermVector.Term>> getTermVector(String id, Map<String, String> fieldMap) throws IOException;

    Map<String, Map<String, List<TermVectorsResponse.TermVector.Term>>> getMultiTermVector(List<String> idList, Map<String, String> fieldMap) throws IOException;

    /**
     * 返回结果计数
     *
     * @param searchSourceBuilder
     * @return
     * @throws IOException
     */
    long count(SearchSourceBuilder searchSourceBuilder) throws IOException;

    /**
     * 判断是否存在
     *
     * @param searchSourceBuilder
     * @return
     * @throws IOException
     */
    boolean exist(SearchSourceBuilder searchSourceBuilder) throws IOException;

    /**
     * 插入更新数据
     *
     * @param entity 实体对象
     */
    String save(T entity);

    /**
     * 插入更新多条数据
     *
     * @param entityList 实体对象列表
     */
    List<String> saveAll(List<T> entityList);


    /**
     * 基于SearchSourceBuilder 搜索
     *
     * @param builder 生成器
     * @return 实体对象列表
     */
    List<T> search(SearchSourceBuilder builder);

    /**
     * 搜索并返回源结果集
     *
     * @param builder
     * @return
     */
    SearchResponse searchWithResponse(SearchSourceBuilder builder);

    /**
     * 更新数据
     *
     * @param entity
     */
    void update(T entity);

    /**
     * 批量更新
     *
     * @param queryBuilder
     * @param entity
     * @throws IOException
     */
    void updateByQuery(QueryBuilder queryBuilder, T entity) throws IOException;

    /**
     * 批量更新
     *
     * @param queryBuilder
     * @param script
     * @throws IOException
     */
    void updateByQuery(QueryBuilder queryBuilder, Script script) throws IOException;

    /**
     * 通过 id删除
     *
     * @param id id
     */
    void deleteById(String id);

    /**
     * 批量删除
     *
     * @param query
     * @throws IOException
     */
    void deleteAllByQuery(QueryBuilder query) throws IOException;

    /**
     * 批量删除数据
     *
     * @param entityList 实体对象列表
     */
    void deleteAll(List<? extends T> entityList);
}

三、搜索基类

默认实现
注入 restHighLevelClient

public abstract class SimpleElasticsearchRepository<T> implements ElasticsearchCrudRepository<T> {
    private final static Logger log = LoggerFactory.getLogger(SimpleElasticsearchRepository.class);

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    private String indexName;
    private String typeName;

    protected void setIndexName(String indexName) {
        this.indexName = indexName;
    }

    protected void setTypeName(String typeName) {
        this.typeName = typeName;
    }

	...

}

1.分页搜索

 /**
     * 分页搜索
     *
     * @param builder  搜索构造器
     * @param pageable 分页器
     * @return
     */
    @Override
    public Page<T> searchPage(SearchSourceBuilder builder, Pageable pageable) {
        builder.from(pageable.getPageNumber());
        builder.size(pageable.getPageSize());

        List<T> content = new ArrayList<>();
        SearchResponse response = searchWithResponse(builder);
        if (Objects.isNull(response)) {
            return new PageImpl<T>(content, pageable, 0);
        }

        // 分页查询结果
        try {
            content = parseSearchResponse(response);
        } catch (Exception e) {
            String msg = "分页查询获取结果失败";
            log.warn(ErrorUtil.getErrorStack(e, msg));
            return new PageImpl<T>(content, pageable, 0);
        }

        // 分页总数
        long count = response.getHits().totalHits;

        PageImpl<T> page = new PageImpl<>(content, pageable, count);

        return page;
    }

2.搜索单个结果

/**
     * 搜索单个文档
     * 设置size为1
     *
     * @param builder 搜索构造器
     * @return
     */
    @Override
    public Optional<T> searchOne(SearchSourceBuilder builder) {
        if (Objects.isNull(builder)) {
            return Optional.empty();
        }
        builder.size(1);
        List<T> list = search(builder);
        if (CollectionUtils.isEmpty(list)) {
            return Optional.empty();
        }
        return Optional.of(list.get(0));
    }

3.搜索返回结果数量

@Override
    public long count(SearchSourceBuilder searchSourceBuilder) throws IOException {
        CountRequest countRequest = getCountRequest();

        countRequest.source(searchSourceBuilder);

        CountResponse countResponse = restHighLevelClient.count(countRequest, RequestOptions.DEFAULT);
        long count = countResponse.getCount();

        return count;
    }

4.插入或者更新数据

/**
     * 插入数据
     *
     * @param entity 实体对象
     */
    @Override
    public String save(T entity) {
        return upsert(entity);
    }
    
/**
     * 插入或者更新数据
     *
     * @param entity 实体类
     * @return id
     */
    public String upsert(T entity) {
        IndexRequest request = getIndexRequest(entity);

        try {
            IndexResponse res = restHighLevelClient.index(request, RequestOptions.DEFAULT);
            return res.getId();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

5.更新数据

@Override
    public void update(T entity) {
        String id = getId(entity);
        UpdateRequest updateRequest = getUpdateRequest(id, entity);

        try {
            restHighLevelClient.update(updateRequest);
        } catch (IOException ioException) {
            throw new RuntimeException(ioException);
        }
    }

6.批量更新数据

	@Override
    public void updateByQuery(QueryBuilder queryBuilder, T entity) throws IOException {
        UpdateByQueryRequest updateByQueryRequest = getUpdateByQueryRequest(queryBuilder, entity);
        if (Objects.nonNull(updateByQueryRequest.getSearchRequest())) {
            log.debug("elasticsearch [updateByQuery]请求体: " + JSONObject.toJSONString(JSON.parseObject(updateByQueryRequest.getSearchRequest().source().toString()), true));
        }

        restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
    }

四、高亮工具

使用<em></em>标签进行高亮 页面需要对标签进行处理

public abstract class HighlightUtil {

    /**
     * 获取高亮器
     *
     * @param field 高亮字段
     * @return
     */
    public static HighlightBuilder getHighlightBuilder(String field) {
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field(field);
        highlightBuilder.requireFieldMatch(false);
        highlightBuilder.preTags("<em>");
        highlightBuilder.postTags("</em>");

        return highlightBuilder;
    }

    /**
     * 获取高亮器 多字段
     *
     * @param fields 多个高亮字段
     * @return
     */
    public static HighlightBuilder getHighlightBuilder(String... fields) {
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        for (String field : fields) {
            highlightBuilder.field(field);
        }
        highlightBuilder.requireFieldMatch(false);
        highlightBuilder.preTags("<em>");
        highlightBuilder.postTags("</em>");

        return highlightBuilder;
    }

    public static HighlightBuilder getHighlightBuilder() {
        return getHighlightBuilder("content");
    }

    /**
     * 处理结果集替换高亮字段
     *
     * @return
     */
    public static void parseResponseWithHighlight(SearchHit hit, JSONObject jsonObject) {
        Map<String, HighlightField> highlightFieldMap = hit.getHighlightFields();
        for (String key : highlightFieldMap.keySet()) {
            HighlightField highlightField = highlightFieldMap.get(key);
            Text[] fragments = highlightField.fragments();
            // 生成高亮文本
            StringBuilder highlightText = new StringBuilder();
            for (Text fragment : fragments) {
                highlightText.append(fragment);
            }
            // 替换成高亮文本
            jsonObject.put(key, highlightText.toString());
        }
    }
}

五、集成示例

/**
 *
 * @author: colagy
 * 2021-01-13 15:12
 */
@Repository
public class UnstructuredDocumentEsDao extends SimpleElasticsearchRepository<UnstructuredDocumentES> {
    @Override
    @PostConstruct
    public void init() {
        this.setIndexName("odata_unstructured_document");
        this.setTypeName("doc");
    }

    public Page<FileSearch> findAllByFileCodeAndKeyword(String fileCode, String keyword, Pageable pageable) {
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        WildcardQueryBuilder fileCodeQuery = QueryBuilders.wildcardQuery("file_code", fileCode + "*");
        MatchQueryBuilder fileNameQuery = QueryBuilders.matchQuery("file_name", keyword);
        MatchQueryBuilder contentQuery = QueryBuilders.matchQuery("content", keyword);
        boolQueryBuilder.must(fileCodeQuery);
        boolQueryBuilder.should(fileNameQuery);
        boolQueryBuilder.must(contentQuery);

        searchSourceBuilder.query(boolQueryBuilder);
        Page<FileSearch> page = this.searchPage(searchSourceBuilder, pageable);
        return page;
    }

}

1. 实现init()方法

调用setIndexName设置index
调用setTypeName设置doc

	@PostConstruct
    public void init() {
        this.setIndexName("odata_unstructured_document");
        this.setTypeName("doc");
    }