一、背景
最近在做录制回放平台, 因为需要把部分数据存储到ES,因此特地实践和调研了一把,把相关材料记录一下;
elastcishearch 版本:7.14.2
spring boot版本:2.6.13
spring-boot-starter-data-elasticsearch: 2.6.13
版本参考文档 https://docs.spring.io/spring-data/elasticsearch/docs/5.2.0-M1/reference/html/#elasticsearch.clients
二、参考文档
https://learnku.com/docs/elasticsearch73/7.3
https://docs.kilvn.com/elasticsearch/docs/6.html
https://spring.io/projects/spring-data-elasticsearch/
三、接入步骤
3.1 maven配置
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.4.0</version>
<relativePath/>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.12</version>
<scope>compile</scope>
</dependency>
</dependencies>
3.2 启动配置
因为考虑可能要支持多数据源,所以我没有考虑在配置中使用 spring.data.elasticsearch.repositories.enabled=true 的方式,而是自定义了启动配置类,代码如下
1 import org.elasticsearch.client.RestHighLevelClient;
2 import org.springframework.beans.factory.annotation.Qualifier;
3 import org.springframework.beans.factory.annotation.Value;
4 import org.springframework.context.annotation.Bean;
5 import org.springframework.context.annotation.Configuration;
6 import org.springframework.data.elasticsearch.client.ClientConfiguration;
7 import org.springframework.data.elasticsearch.client.RestClients;
8 import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
9 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
10 import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
11 import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
12
13
14 @Configuration
15 public class RestClientConfig extends AbstractElasticsearchConfiguration {
16
17
18 @Value("${elasticsearch.url}")
19 private String elasticsearchUrl;
20
21 @Value("${elasticsearch.username}")
22 private String username;
23
24 @Value("${elasticsearch.password}")
25 private String password;
26
27
28 @Bean("esClient")
29 @Override
30 public RestHighLevelClient elasticsearchClient() {
31 final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
32 .connectedTo(elasticsearchUrl)
33 .withBasicAuth(username, password)
34 .build();
35 return RestClients.create(clientConfiguration).rest();
36 }
37
38 @Override
39 @Bean(name = {"elasticSearchRestTemplate" })
40 public ElasticsearchRestTemplate elasticsearchOperations(ElasticsearchConverter elasticsearchConverter,
41 @Qualifier("esClient") RestHighLevelClient elasticsearchClient) {
42 return new ElasticsearchRestTemplate(elasticsearchClient, elasticsearchConverter);
43 }
44
45 }
以下配置可以放入到application.properties里面即可
elasticsearch.url=xxxx
elasticsearch.username=xxxx
elasticsearch.password=xxx
3.3 创建索引&定义实体对象
elasticsearch索引的创建,可以手动创建,也可以自动创建;我们可以利用spring-data-elasticsearch的自动创建的能力来构建索引
package com.example.es.entity;
import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import org.springframework.data.elasticsearch.annotations.Setting;
import java.util.Date;
/**
* @author peng.hu1
* @Date 2023/7/13 15:06
*/
@Data
@Setting(shards = 3)
@Document(indexName = "record", createIndex = true)
public class RecordEntity {
/**
* 主键
*/
@Id
private String id;
@Field(name = "api", type = FieldType.Keyword)
private String api;
/**
* 创建时间
*/
@Field(value = "create_time")
private Date createTime;
/**
* 应用名
*/
@Field(value = "app_name", type = FieldType.Keyword)
private String appName;
/**
* 环境信息
*/
@Field(value = "env", type = FieldType.Keyword)
private String env;
/**
* 机器IP
*/
@Field(value = "host", type = FieldType.Keyword)
private String host;
/**
* 链路追踪ID
*/
@Field(value = "trace_id", type = FieldType.Keyword)
private String traceId;
/**
* 主调用
*/
@Field(value = "entrance_desc", type = FieldType.Keyword)
private String entranceDesc;
/**
* 记录序列化信息
*/
@Field(value = "wrapper_record", type = FieldType.Keyword, index = false)
private String wrapperRecord;
/**
* 请求参数JSON
*/
@Field(type = FieldType.Text, index=false)
private String request;
/**
* 返回值JSON
*/
@Field(type = FieldType.Text, index=false)
private String response;
}
定义实体的时候,有几个关键注解需要说明一下,
@Document(indexName = "record", createIndex = true) 其中indexName为索引名,createIndex=true表示默认在插入的时候,会创建索引;索引是包含mapping 和 setting 的,mapping的配置是从@Field注解生成的; @Setting注解用于设置分片相关物理属性;
public @interface Field {
FieldType type() default FieldType.Auto; //自动检测属性的类型,可以根据实际情况自己设置
FieldIndex index() default FieldIndex.analyzed; //默认情况下分词,一般默认分词就好,除非这个字段你确定查询时不会用到
DateFormat format() default DateFormat.none; //时间类型的格式化
String pattern() default "";
boolean store() default false; //默认情况下不存储原文
String searchAnalyzer() default ""; //指定字段搜索时使用的分词器
String indexAnalyzer() default ""; //指定字段建立索引时指定的分词器
String[] ignoreFields() default {}; //如果某个字段需要被忽略
boolean includeInParent() default false;
}
FieldType.Keyword 跟 FieldType.Text 都可以运用到字符串,最大的区别在于Text是会分词建立倒排索引的,这个按需选择类型就好;如果只是当做db来用,默认强制设置成keyWord类型更合适;
3.4 对索引对象的增删改查
定义接口
package com.example.es.service;
import com.example.es.entity.RecordEntity;
import com.example.es.model.Page;
import com.example.es.model.Search;
import java.util.List;
/**
* @author peng.hu1
* @Date 2023/8/17 16:22
*/
public interface RecordEntityService {
public void save(RecordEntity entity);
public void batchSave(List<RecordEntity> entityList);
public Page<RecordEntity> search(Search search);
public void remove(RecordEntity entity);
}
接口实现
package com.example.es.impl;
import com.example.es.entity.RecordEntity;
import com.example.es.model.Page;
import com.example.es.model.Search;
import com.example.es.service.RecordEntityService;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author peng.hu1
* @Date 2023/8/17 16:25
*/
@Component
public class RecordEntityServiceImpl implements RecordEntityService {
@Resource(name = "elasticSearchRestTemplate")
ElasticsearchRestTemplate elasticsearchRestTemplate;
@Override
public void save(RecordEntity entity) {
elasticsearchRestTemplate.save(entity);
}
@Override
public void batchSave(List<RecordEntity> entityList) {
elasticsearchRestTemplate.save(entityList);
}
@Override
public Page<RecordEntity> search(Search r) {
NativeSearchQueryBuilder query = new NativeSearchQueryBuilder();
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
query.withQuery(queryBuilder);
if(StringUtils.hasLength(r.getAppName())) {
queryBuilder.filter(QueryBuilders.matchPhraseQuery("app_name", r.getAppName()));
}
if(StringUtils.hasLength(r.getEnv())) {
queryBuilder.filter(QueryBuilders.matchPhraseQuery("env", r.getEnv()));
}
//分页 & 排序
PageRequest pageRequest = PageRequest.of(r.getPage()-1, r.getPageSize());
query.withQuery(queryBuilder);
query.withPageable(pageRequest);
query.withSorts(SortBuilders.fieldSort("create_time").order(SortOrder.DESC));
SearchHits<RecordEntity> searchHits = elasticsearchRestTemplate.search(query.build(), RecordEntity.class);
Page<RecordEntity> res = new Page<>();
res.setPage(r.getPage());
res.setPageSize(r.getPageSize());
res.setTotal(searchHits.getTotalHits());
res.setData(searchHits.stream().map(SearchHit::getContent).collect(Collectors.toList()));
return res;
}
@Override
public void remove(RecordEntity entity) {
elasticsearchRestTemplate.delete(entity);
}
}
至此接入流程就全部结束了