springboot整合elasticsearch及热更新字典及同义词

最新推荐文章于 2024-08-16 09:49:09 发布

futhead

最新推荐文章于 2024-08-16 09:49:09 发布

阅读量1.9k

点赞数

文章标签： elasticsearch springboot 同义词 DSL 字典热更新

本文链接：https://blog.csdn.net/fuhtead/article/details/100862628

版权

整合

dao层ItemRepository

package com.futhead.es.dao;

import com.futhead.es.model.Item;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;

import java.util.List;


public interface ItemRepository extends ElasticsearchRepository<Item, Long> {

    List<Item> findByPriceBetween(double price1, double price2);

}

model Item

package com.futhead.es.model;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;

@Data
@AllArgsConstructor
@NoArgsConstructor
@Document(indexName = "item",type = "docs", shards = 1, replicas = 0)
public class Item {

    @Id
    private Long id;

    @Field(type = FieldType.Text, analyzer = "ik_max_word")
    private String title; //标题

    @Field(type = FieldType.Keyword)
    private String category;// 分类

    @Field(type = FieldType.Keyword)
    private String brand; // 品牌

    @Field(type = FieldType.Double)
    private Double price; // 价格

    @Field(index = false, type = FieldType.Keyword)
    private String images; // 图片地址

}

pom.xml

<dependencies>
	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter</artifactId>
	</dependency>

	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
	</dependency>

	<dependency>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-test</artifactId>
		<scope>test</scope>
	</dependency>

	<dependency>
		<groupId>org.projectlombok</groupId>
		<artifactId>lombok</artifactId>
		<version>1.18.8</version>
		<scope>provided</scope>
	</dependency>
</dependencies>

es配置

spring.data.elasticsearch.cluster-name=my-application
spring.data.elasticsearch.cluster-nodes=localhost:9300

测试用例

import com.futhead.es.Application;
import com.futhead.es.dao.ItemRepository;
import com.futhead.es.model.Item;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilter;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.ArrayList;
import java.util.List;


@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class)
public class EsDemoApplicationTests {

    @Autowired
    private ElasticsearchTemplate elasticsearchTemplate;

    @Autowired
    private ItemRepository itemRepository;

    @Test
    public void testCreateIndex() {
        elasticsearchTemplate.createIndex(Item.class);
    }

    /**
     * @Description:定义批量新增方法
     * @Author: https://blog.csdn.net/chen_2890
     */
    @Test
    public void insertList() {
        List<Item> list = new ArrayList<>();
        list.add(new Item(1L, "坚果手机R1", " 手机", "锤子", 3699.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(2L, "华为META10", " 手机", "华为", 4499.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(3L, "小米手机7", "手机", "小米", 3299.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(4L, "坚果手机R1", "手机", "锤子", 3699.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(5L, "华为META10", "手机", "华为", 4499.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(6L, "小米Mix2S", "手机", "小米", 4299.00, "http://image.baidu.com/13123.jpg"));
        list.add(new Item(7L, "荣耀V10", "手机", "华为", 2799.00, "http://image.baidu.com/13123.jpg"));

        // 接收对象集合，实现批量新增
        itemRepository.saveAll(list);
    }


    /**
     * @Description:按照价格区间查询
     * @Author: https://blog.csdn.net/chen_2890
     */
    @Test
    public void queryByPriceBetween(){
        List<Item> list = this.itemRepository.findByPriceBetween(2000.00, 3500.00);
        for (Item item : list) {
            System.out.println("item = " + item);
        }
    }


    @Test
    public void testTermQuery(){
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        builder.withQuery(QueryBuilders.termQuery("price",998.0));
        // 查找
        Page<Item> page = this.itemRepository.search(builder.build());

        for(Item item:page){
            System.out.println(item);
        }
    }
    /**
     * @Description:布尔查询
     * @Author: https://blog.csdn.net/chen_2890
     */
    @Test
    public void testBooleanQuery(){
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();

//        builder.withQuery(
//                QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("title","华为"))
//                        .must(QueryBuilders.matchQuery("brand","华为"))
//        );

        builder.withQuery(
                QueryBuilders.boolQuery().should(QueryBuilders.matchQuery("title","荣耀")).boost(2.0f)
                        .should(QueryBuilders.matchQuery("brand","锤子"))
        );

        // 查找
        Page<Item> page = this.itemRepository.search(builder.build());
        for(Item item:page){
            System.out.println(item);
        }
    }


    /**
     * @Description:嵌套聚合，求平均值
     * @Author: https://blog.csdn.net/chen_2890
     */
    @Test
    public void testSubAgg(){
        NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
        // 不查询任何结果
        queryBuilder.withSourceFilter(new FetchSourceFilter(new String[]{""}, null));
        // 1、添加一个新的聚合，聚合类型为terms，聚合名称为brands，聚合字段为brand
        queryBuilder.addAggregation(
                AggregationBuilders.terms("brands").field("brand")
                        .subAggregation(AggregationBuilders.avg("priceAvg").field("price")) // 在品牌聚合桶内进行嵌套聚合，求平均值
        );
        // 2、查询,需要把结果强转为AggregatedPage类型
        AggregatedPage<Item> aggPage = (AggregatedPage<Item>) this.itemRepository.search(queryBuilder.build());
        // 3、解析
        // 3.1、从结果中取出名为brands的那个聚合，
        // 因为是利用String类型字段来进行的term聚合，所以结果要强转为StringTerm类型
        StringTerms agg = (StringTerms) aggPage.getAggregation("brands");
        // 3.2、获取桶
        List<StringTerms.Bucket> buckets = agg.getBuckets();
        // 3.3、遍历
        for (StringTerms.Bucket bucket : buckets) {
            // 3.4、获取桶中的key，即品牌名称  3.5、获取桶中的文档数量
            System.out.println(bucket.getKeyAsString() + "，共" + bucket.getDocCount() + "台");

            // 3.6.获取子聚合结果：
            InternalAvg avg = (InternalAvg) bucket.getAggregations().asMap().get("priceAvg");
            System.out.println("平均售价：" + avg.getValue());
        }
    }
}

打印es搜索语句（DSL）

@Test
public void testMathQuery(){
	// 创建对象
	NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder();
	// 在queryBuilder对象中自定义查询
	//matchQuery:底层就是使用的termQuery
	queryBuilder.withQuery(QueryBuilders.matchQuery("title","坚果"));
	//查询，search 默认就是分页查找
	SearchQuery searchQuery = queryBuilder.build();

	//打印查询语句;
	System.out.println("拼接的查询请求======");
	System.out.println(searchQuery.getQuery().toString());

	Page<Item> page = this.itemRepository.search(searchQuery);
	//获取数据
	long totalElements = page.getTotalElements();
	System.out.println("获取的总条数:"+totalElements);
}

简单的增加权重（boost）

@Test
public void testBooleanQuery(){
    NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
    builder.withQuery(
            QueryBuilders.boolQuery().should(QueryBuilders.matchQuery("title","荣耀")).boost(2.0f)
                    .should(QueryBuilders.matchQuery("brand","锤子"))
    );
    Page<Item> page = this.itemRepository.search(builder.build());
    for(Item item:page){
        System.out.println(item);
    }
}

ik分词器，支持热更新

github地址：https://github.com/medcl/elasticsearch-analysis-ik

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
	<comment>IK Analyzer 扩展配置</comment>
	<!--用户可以在这里配置自己的扩展字典 -->
	<entry key="ext_dict">custom/mydict.dic;custom/single_word_low_freq.dic</entry>
	 <!--用户可以在这里配置自己的扩展停止词字典-->
	<entry key="ext_stopwords">custom/ext_stopword.dic</entry>
 	<!--用户可以在这里配置远程扩展字典 -->
	<entry key="remote_ext_dict">location</entry>
 	<!--用户可以在这里配置远程扩展停止词字典-->
	<entry key="remote_ext_stopwords">http://xxx.com/xxx.dic</entry>
</properties>

Ps. 该 http 请求需要返回两个头部(header)，一个是 Last-Modified，一个是 ETag，这两者都是字符串类型，只要有一个发生变化，该插件就会去抓取新的分词进而更新词库。

同义词、近义词热更新

获取插件

github地址：https://github.com/bells/elasticsearch-analysis-dynamic-synonym

使用方法同ik分词器

Ps. release版本比较少，一般需要根据自己的elasticsearch版本编译

eg. 我们使用的事es6.5.0

git clone https://github.com/bells/elasticsearch-analysis-dynamic-synonym.git

在提交分支记录中找到和自己使用的elasticsearch版本最近的分支，比如，我们用的6.5.0,提交记录支持到6.3.1

git checkout dbe8ebeb6b92d9b403acb76020ea7d64b39abcc8

修改pom.xml文件中的version为6.5.0

编译：mvn clean package

在target/release目录下就能看到最终输出的插件压缩包。

ps. 6.5.0中日志升级了，直接编译不能通过，需要修改

//import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.logging.Loggers;

//private static Logger logger = ESLoggerFactory.getLogger("dynamic-synonym");
private static Logger logger = Loggers.getLogger(String.class,"dynamic-synonym");

使用

近义词的两种形式：

# synonyms.txt
西红柿 圣女果 番茄 => 洋柿子    # 将 => 左边的词在分词的时候划归为右边的词,检索时只能检索右边的词
西红柿, 圣女果, 番茄    # 三个词同意，检索任何一个也能搜索到另外两个

参考资料

https://blog.csdn.net/chen_2890/article/details/83895646