全文检索服务 ElasticSearch其他相关:
Spring Boot整合ElasticSearch
1.1.ES客户端
ES提供多种不同的客户端:
1、TransportClient
ES提供的传统客户端,官方计划8.0版本删除此客户端。
2、RestClient
RestClient是官方推荐使用的,它包括两种:REST Low Level Client和 REST High Level Client。ES在6.0之后提供REST High Level Client, 两种客户端官方更推荐使用 REST High Level Client,不过当前它还处于完善中,有些功能还没有。
1.2.搭建工程
1.2.1.pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.2.RELEASE</version>
</parent>
<groupId>com.zzcsy</groupId>
<artifactId>springboot_elasticsearch</artifactId>
<version>1.0-SNAPSHOT</version>
<!-- 修改elasticsearch的版本 -->
<properties>
<elasticsearch.version>6.2.3</elasticsearch.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
</dependencies>
</project>
1.2.2.application.yml
spring:
elasticsearch:
rest:
uris:
- http://192.168.204.132:9200
1.2.3.app
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ElasticsearchApp {
public static void main(String[] args) {
SpringApplication.run(ElasticsearchApp.class, args);
}
}
1.3.索引管理
1.3.1.创建索引库
1.3.1.1.api
创建索引库:
PUT /java06
{
"settings":{
"number_of_shards" : 2,
"number_of_replicas" : 0
}
}
创建映射:
POST /java06/course/_mapping
{
"_source": {
"excludes":["description"]
},
"properties": {
"name": {
"type": "text",
"analyzer":"ik_max_word",
"search_analyzer":"ik_smart"
},
"description": {
"type": "text",
"analyzer":"ik_max_word",
"search_analyzer":"ik_smart"
},
"studymodel": {
"type": "keyword"
},
"price": {
"type": "float"
},
"pic":{
"type":"text",
"index":false
}
}
}
1.3.1.2.Java Client
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.IndicesClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest(classes = {ElasticsearchApp.class})
public class IndexWriterTest {
@Autowired
private RestHighLevelClient restHighLevelClient;
//创建索引库
@Test
public void testCreateIndex() throws IOException {
//创建“创建索引请求”对象,并设置索引名称
CreateIndexRequest createIndexRequest = new CreateIndexRequest("java06");
//设置索引参数
createIndexRequest.settings("{\n" +
" \"number_of_shards\" : 2,\n" +
" \"number_of_replicas\" : 0\n" +
" }", XContentType.JSON);
createIndexRequest.mapping("course", "{\r\n" +
" \"_source\": {\r\n" +
" \"excludes\":[\"description\"]\r\n" +
" }, \r\n" +
" \"properties\": {\r\n" +
" \"name\": {\r\n" +
" \"type\": \"text\",\r\n" +
" \"analyzer\":\"ik_max_word\",\r\n" +
" \"search_analyzer\":\"ik_smart\"\r\n" +
" },\r\n" +
" \"description\": {\r\n" +
" \"type\": \"text\",\r\n" +
" \"analyzer\":\"ik_max_word\",\r\n" +
" \"search_analyzer\":\"ik_smart\"\r\n" +
" },\r\n" +
" \"studymodel\": {\r\n" +
" \"type\": \"keyword\"\r\n" +
" },\r\n" +
" \"price\": {\r\n" +
" \"type\": \"float\"\r\n" +
" },\r\n" +
" }\r\n" +
"}", XContentType.JSON);
//创建索引操作客户端
IndicesClient indices = restHighLevelClient.indices();
//创建响应对象
CreateIndexResponse createIndexResponse =
indices.create(createIndexRequest);
//得到响应结果
boolean acknowledged = createIndexResponse.isAcknowledged();
System.out.println(acknowledged);
}
}
1.3.2.删除索引库
1.3.2.1.api
DELETE /java06
1.3.2.2.java client
//删除索引库
@Test
public void testDeleteIndex() throws IOException {
//创建“删除索引请求”对象
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("java06");
//创建索引操作客户端
IndicesClient indices = restHighLevelClient.indices();
//创建响应对象
DeleteIndexResponse deleteIndexResponse =
indices.delete(deleteIndexRequest);
//得到响应结果
boolean acknowledged = deleteIndexResponse.isAcknowledged();
System.out.println(acknowledged);
}
1.3.2.添加文档
1.3.2.1.api
POST /java06/course/1
{
"name":"spring cloud实战",
"description":"本课程主要从四个章节进行讲解: 1.微服务架构入门 2.spring cloud 基础入门 3.实战Spring Boot 4.注册中心eureka。",
"studymodel":"201001",
"price":5.6
}
1.3.2.2.java client
//添加文档
@Test
public void testAddDocument() throws IOException {
//创建“索引请求”对象:索引当动词
IndexRequest indexRequest = new IndexRequest("java06", "course", "1");
indexRequest.source("{\n" +
" \"name\":\"spring cloud实战\",\n" +
" \"description\":\"本课程主要从四个章节进行讲解: 1.微服务架构入门 " +
"2.spring cloud 基础入门 3.实战Spring Boot 4.注册中心nacos。\",\n" +
" \"studymodel\":\"201001\",\n" +
" \"price\":5.6\n" +
"}", XContentType.JSON);
IndexResponse indexResponse =
restHighLevelClient.index(indexRequest);
System.out.println(indexResponse.toString());
}
1.3.3.批量添加文档
支持在一次API调用中,对不同的索引进行操作。支持四种类型的操作:index、create、update、delete。
- 语法:
POST /_bulk
{ action: { metadata }}
{ requestbody }\n
{ action: { metadata }}
{ requestbody }\n
...
1.3.3.1.api
POST /_bulk
{"index":{"_index":"java06","_type":"course"}}
{"name":"php实战","description":"php谁都不服","studymodel":"201001","price":"5.6"}
{"index":{"_index":"java06","_type":"course"}}
{"name":"net实战","description":"net从入门到放弃","studymodel":"201001","price":"7.6"}
1.3.3.2.java client
@Test
public void testBulkAddDocument() throws IOException {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(new IndexRequest("java06", "course").source("{...}",
XContentType.JSON));
bulkRequest.add(new IndexRequest("java06", "course").source("{...}",
XContentType.JSON));
BulkResponse bulkResponse =
restHighLevelClient.bulk(bulkRequest);
System.out.println(bulkResponse.hasFailures());
}
1.3.4.修改文档
1.3.4.1.api
PUT /java06/course/1
{
"price":66.6
}
1.3.4.2.java client
//更新文档
@Test
public void testUpdateDocument() throws IOException {
UpdateRequest updateRequest = new UpdateRequest("java06", "course", "1");
updateRequest.doc("{\n" +
" \"price\":7.6\n" +
"}", XContentType.JSON);
UpdateResponse updateResponse =
restHighLevelClient.update(updateRequest);
System.out.println(updateResponse.getResult());
}
1.3.5.删除文档
1.3.5.1.api
DELETE /java06/coures/1
1.3.4.2.java client
//根据id删除文档
@Test
public void testDelDocument() throws IOException {
//删除请求对象
DeleteRequest deleteRequest = new DeleteRequest("java06","course","1");
//响应对象
DeleteResponse deleteResponse =
restHighLevelClient.delete(deleteRequest);
System.out.println(deleteResponse.getResult());
}
1.4.文档搜索
1.4.1.准备环境
向索引库中插入以下数据:
PUT /java06/course/1
{
"name": "Bootstrap开发",
"description": "Bootstrap是由Twitter推出的一个前台页面开发css框架,是一个非常流行的开发框架,此框架集成了多种页面效果。此开发框架包含了大量的CSS、JS程序代码,可以帮助开发者(尤其是不擅长css页面开发的程序人员)轻松的实现一个css,不受浏览器限制的精美界面css效果。",
"studymodel": "201002",
"price":38.6,
"pic":"group1/M00/00/00/wKhlQFs6RCeAY0pHAAJx5ZjNDEM428.jpg"
}
PUT /java06/course/2
{
"name": "java编程基础",
"description": "java语言是世界第一编程语言,在软件开发领域使用人数最多。",
"studymodel": "201001",
"price":68.6,
"pic":"group1/M00/00/00/wKhlQFs6RCeAY0pHAAJx5ZjNDEM428.jpg"
}
PUT /java06/course/3
{
"name": "spring开发基础",
"description": "spring 在java领域非常流行,java程序员都在用。",
"studymodel": "201001",
"price":88.6,
"pic":"group1/M00/00/00/wKhlQFs6RCeAY0pHAAJx5ZjNDEM428.jpg"
}
1.4.2.简单搜索
简单搜索就是通过url进行查询,以get方式请求ES。
语法:
GET /index_name/type_name/doc_id
GET [/index_name/type_name/]_search[?parameter_name=parameter_value&…]
例如:
GET /java06/course/_search?q=name:spring&sort=price:desc
注意:
如果查询条件复杂,很难构建搜索条件 ,生产环境中很少使用。
例如:要求搜索条件为商品名称包含手机,价格在 1000~5000之间,销量在每月 500 以上,根据价格升序排列,分页查询第二页,每页 40 条数据:?q=xxxx:xxx&range=xxx:xxx:xxx&aggs&sort&from&size
1.4.2.1.api
GET /java06/course/1
8.4.2.2.java client
//查询文档
@Test
public void getDoc() throws IOException {
GetRequest getRequest = new GetRequest("java06","course","1");
GetResponse getResponse = restHighLevelClient.get(getRequest);
boolean exists = getResponse.isExists();
System.out.println(exists);
String source = getResponse.getSourceAsString();
System.out.println(source);
}
1.4.3.DSL搜索
DSL(Domain Specific Language)是ES提出的基于json的搜索方式,在搜索时传入特定的json格式的数据来完成不同的搜索需求,DSL比URI搜索方式功能强大,在项目中建议使用DSL方式来完成搜索。
语法:
GET /index_name/type_name/_search
{
“commond”:{
“parameter_name” : “parameter_value”
}
}
1.4.3.1.match_all查询
1.4.3.1.1.api
GET /java06/course/_search
{
"query" : {
"match_all" : {}
}
}
1.4.3.1.2.java client
import com.zzcsy.ElasticsearchApp;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.io.IOException;
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest(classes = {ElasticsearchApp.class})
public class IndexReaderTest {
@Autowired
private RestHighLevelClient restHighLevelClient;
private SearchRequest searchRequest;
private SearchResponse searchResponse;
@Before
public void init(){
searchRequest = new SearchRequest();
searchRequest.indices("java06");
searchRequest.types("course");
}
@Test
public void testMatchAll() throws IOException {
//2、创建 search请求对象
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("java06");
searchRequest.types("course");
//3、创建 参数构造器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery());
//4、设置请求参数
searchRequest.source(searchSourceBuilder);
//1、调用search方法
SearchResponse searchResponse = restHighLevelClient.search(searchRequest);
SearchHits searchHits = searchResponse.getHits();
long totalHits = searchHits.getTotalHits();
System.out.println("共搜索到"+totalHits+"条文档");
SearchHit[] hits = searchHits.getHits();
for (SearchHit hit : hits) {
System.out.println(hit.getSourceAsString());
}
}
@After
public void show(){
SearchHits searchHits = searchResponse.getHits();
long totalHits = searchHits.getTotalHits();
System.out.println("共搜索到"+totalHits+"条文档");
SearchHit[] hits = searchHits.getHits();
for (SearchHit hit : hits) {
System.out.println(hit.getSourceAsString());
}
}
}
1.3.3.2.分页查询
1.3.3.2.1.api
GET /java06/course/_search
{
"query" : { "match_all" : {} },
"from" : 1, # 从第几条数据开始查询,从0开始计数
"size" : 3, # 查询多少数据
"sort" : [
{ "price" : "asc" }
]
}
1.3.3.2.2.java client
//分页查询
@Test
public void testSearchPage() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery());
searchSourceBuilder.from(1);
searchSourceBuilder.size(5);
searchSourceBuilder.sort("price", SortOrder.ASC);
// 设置搜索源
searchRequest.source(searchSourceBuilder);
// 执行搜索
searchResponse = restHighLevelClient.search(searchRequest);
}
1.3.3.4.match查询
match Query即全文检索,它的搜索方式是先将搜索字符串分词,再使用各各词条从索引中搜索。
1.3.3.4.1.api
query:搜索的关键字
operator:or 表示 只要有一个词在文档中出现则就符合条件,and表示每个词都在文档中出现则才符合条件。
1、基本使用:
GET /java06/course/_search
{
"query" : {
"match" : {
"name": {
"query": "spring开发"
}
}
}
}
2、operator:
GET /java06/course/_search
{
"query" : {
"match" : {
"name": {
"query": "spring开发",
"operator": "and"
}
}
}
}
上边的搜索的执行过程是:
1、将“spring开发”分词,分为spring、开发两个词
2、再使用spring和开发两个词去匹配索引中搜索。
3、由于设置了operator为and,必须匹配两个词成功时才返回该文档。
1.3.3.4.2 java client
@Test
public void testMatchQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("name", "spring开
发").operator(Operator.AND));
// 设置搜索源
searchRequest.source(searchSourceBuilder);
// 执行搜索
searchResponse = restHighLevelClient.search(searchRequest);
}
1.3.3.5.multi_match查询
matchQuery是在一个field中去匹配,multiQuery是拿关键字去多个Field中匹配。
1.3.3.5.1.api
1、基本使用
例子:关键字 “开发”去匹配name 和description字段
GET /java06/course/_search
{
"query": {
"multi_match": {
"query": "开发",
"fields": ["name","description"]
}
}
}
注意:此搜索操作适合构建复杂查询条件,生产环境常用。
1.3.3.5.2.java client
@Test
public void testMultiMatchQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.multiMatchQuery("开发","name","description"));
// 设置搜索源
searchRequest.source(searchSourceBuilder);
// 执行搜索
searchResponse = restHighLevelClient.search(searchRequest);
}
1.3.3.6.bool查询
布尔查询对应于Lucene的BooleanQuery查询,实现将多个查询组合起来。
参数:
must:表示必须,多个查询条件必须都满足。(通常使用must)
should:表示或者,多个查询条件只要有一个满足即可。
must_not:表示非。
1.3.3.6.1.api
例如:查询name包括“开发”并且价格区间是1-100的文档
GET /java06/course/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "开发"
}
},
{
"range": {
"price": {
"gte": 50,
"lte": 100
}
}
}
]
}
}
}
1.3.3.6.2.java client
@Test
public void testBooleanMatch() throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//json条件
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name","开发"));
boolQueryBuilder.must(QueryBuilders.rangeQuery("price").gte("50").lte(100));
searchSourceBuilder.query(boolQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest);
}
1.3.3.7.filter查询
过滤查询。此操作实际上就是 query DSL 的补充语法。过滤的时候,不进行任何的匹配分数计算,相对于 query 来说,filter 相对效率较高。Query 要计算搜索匹配相关度分数。Query更加适合复杂的条件搜索。
1.3.3.7.1.api
如:使用bool查询,搜索 name中包含 "开发"的数据,且price在 10~100 之间
1、不使用 filter, name和price需要计算相关度分数:
GET /java06/course/_search
{
"query": {
"bool" : {
"must":[
{
"match": {
"name": "开发"
}
},
{
"range": {# 范围, 字段的数据必须满足某范围才有结果。
"price": {
"gte": 10, # 比较符号 lt gt lte gte
"lte": 100
}
}
}
]
}
}
}
2、使用 filter, price不需要计算相关度分数:
GET /java06/course/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "开发"
}
}
],
"filter": {# 过滤,在已有的搜索结果中进行过滤,满足条件的返回。
"range": {
"price": {
"gte": 1,
"lte": 100
}
}
}
}
}
}
1.3.3.7.2.java client
@Test
public void testFilterQuery() throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name","开发"));
boolQueryBuilder.filter(QueryBuilders.rangeQuery("price").gte(10).lte(100))
searchSourceBuilder.query(boolQueryBuilder);
searchRequest.source(searchSourceBuilder);
searchResponse = restHighLevelClient.search(searchRequest);
}
1.3.3.8.highlight查询
高亮显示:高亮不是搜索条件,是显示逻辑,在搜索的时候,经常需要对搜索关键字实现高亮显示。
1.3.3.8.1.api
例如:
GET /java06/course/_search
{
"query": {
"match": {
"name": "开发"
}
},
"highlight": {
"pre_tags": ["<font color='red'>"],
"post_tags": ["</font>"],
"fields": {"name": {}}
}
}
1.3.3.8.2.java clent
1、查询:
@Test
public void testHighLightQuery() throws Exception {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("name", "spring"));
//设置高亮
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.preTags("<font color='red'>");
highlightBuilder.postTags("</font>");
highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
searchSourceBuilder.highlighter(highlightBuilder);
searchRequest.source(searchSourceBuilder);
searchResponse = restHighLevelClient.search(searchRequest);
}
2、遍历:
@After
public void displayDoc() {
SearchHits searchHits = searchResponse.getHits();
long totalHits = searchHits.getTotalHits();
System.out.println("共搜索到" + totalHits + "条文档");
SearchHit[] hits = searchHits.getHits();
for (int i = 0; i < hits.length; i++) {
SearchHit hit = hits[i];
String id = hit.getId();
System.out.println("id:" + id);
String source = hit.getSourceAsString();
System.out.println(source);
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
if (highlightFields != null) {
HighlightField highlightField = highlightFields.get("name");
Text[] fragments = highlightField.getFragments();
System.out.println("高亮字段:" + fragments[0].toString());
}
}
}