Spring Boot 整合 ES
前期准备
- 依赖文件
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<junit.version>4.12</junit.version>
<nacos.context>2.1.0-RC</nacos.context>
<!-- <swagger.ui>2.9.2</swagger.ui> -->
<swagger.ui>3.0.0</swagger.ui>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<log4j.version>1.2.17</log4j.version>
<lombok.version>1.18.26</lombok.version>
<mysql.version>5.1.47</mysql.version>
<druid.version>1.1.16</druid.version>
<druid.spring.boot.starter.version>1.1.10</druid.spring.boot.starter.version>
<mapper.version>4.1.5</mapper.version>
<mybatis.spring.boot.version>1.3.0</mybatis.spring.boot.version>
<mysql.connector.version>5.1.47</mysql.connector.version>
<hutool.version>5.2.3</hutool.version>
<mybatis.plus.boot.starter.version>3.2.0</mybatis.plus.boot.starter.version>
<guava.version>23.0</guava.version>
<canal.client.version>1.1.0</canal.client.version>
<redission.version>3.19.1</redission.version>
<elasticsearch.version>7.8.0</elasticsearch.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2-extension</artifactId>
<version>2.0.12</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch</groupId>-->
<!-- <artifactId>elasticsearch</artifactId>-->
<!-- <version>${elasticsearch.version}</version>-->
<!-- </dependency>-->
<!-- <!– elasticsearch 的客户端 –>-->
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
<!-- <version>${elasticsearch.version}</version>-->
<!-- </dependency>-->
<!--基于AMQP协议的消息中间件框架-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<!-- SpringBoot通用依赖模块 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- 通用基础配置 -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<optional>true</optional>
</dependency>
<!-- swagger2 -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-boot-starter</artifactId>
<version>${swagger.ui}</version>
</dependency>
<!-- lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
<!-- 通用基础配置junit/devtools/test/log4j/lombok/hutool -->
<!-- hutool -->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>${hutool.version}</version>
</dependency>
<dependency>
<groupId>com.vaadin.external.google</groupId>
<artifactId>android-json</artifactId>
<version>0.0.20131108.vaadin1</version>
<scope>compile</scope>
</dependency>
</dependencies>
- 配置文件
server:
port: 8080
spring:
application:
name: rabbit-mq-learn
swagger2:
enabled: true
elasticsearch:
rest:
uris: http://43.138.25.182:9200
connection-timeout: 5s
- PO
这里
@Document
注解设置class
在ES
中的索引和分片等信息
@Field
设置字段的类型信息,索引状态等,用来创建映射使用的的信息
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Document(indexName = "user", shards = 3, replicas = 1)
public class User implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 使用@Id注解声明该字段为文档的唯一标识,使用@Field注解声明该字段为关键字字段,并指定字段不可分词
*/
@Id
@Field(type = FieldType.Keyword,fielddata = true)
private String id;
/**
* 姓名可以进行分册查询
*/
@Field(type = FieldType.Text)
private String name;
/**
* 性别: 1,男 2,女
*/
@Field(type = FieldType.Integer)
private Integer age;
/**
* 住址可进行分词查询
*/
@Field(type = FieldType.Text)
private String address;
/**
* 电子邮箱地址不可分词
*/
@Field(type = FieldType.Keyword)
private String email;
/**
* 电话不可分词
*/
@Field(type = FieldType.Keyword)
private String tel;
/**
* 日期类型
*/
@Field(type = FieldType.Date)
private Date birthday;
}
- DAO
这里我们就可以和使用
mybatis
一样去操作es
了
@Repository
public interface UserDao extends ElasticsearchRepository<User,String> {
}
索引操作
索引的创建,删除, 查询所有索引
@SpringBootTest
@RunWith(SpringRunner.class)
class ESIndexOpreationTest {
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Autowired
private RestHighLevelClient restHighLevelClient;
/**
* 判断是否存在索引 存在删除再次创建,不存在直接创建
*/
@Test
void indexOperation() {
// 获取索引操作对象
IndexOperations indexOperations = elasticsearchRestTemplate.indexOps(User.class);
// 判断索引是否存在
boolean exists = indexOperations.exists();
if (exists){
System.out.println("exists:"+exists);
//删除索引
System.out.println("delete:"+indexOperations.delete());
}
//创建索引
System.out.println("create:"+indexOperations.create());
Document mapping = indexOperations.createMapping();
System.out.println(mapping);
}
/**
* 获取所有 index
* @throws IOException
*/
@Test
void getAllIndex() throws IOException {
GetIndexRequest request = new GetIndexRequest();
// 设置匹配所有索引的模式(默认也是匹配所有,这里为了明确指出)
request.indices("*");
GetIndexResponse getIndexResponse = restHighLevelClient.indices().get(request, RequestOptions.DEFAULT);
String[] indices = getIndexResponse.getIndices();
for (int i = 0; i < indices.length; i++) {
System.out.println(indices[i]);
}
}
}
文档操作
文档添加
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
/**
* 测试添加文档
*/
@Test
void addDocument() {
String name = "whn";
//String id = UUID.nameUUIDFromBytes(name.getBytes()).toString();
String id = "whn123456";
User user = User.builder()
.id(id).address("南京鼓楼")
.age(18).email("147258369@qq.com")
.tel("1234567890")
.birthday(new Date())
.name(name).build();
User save = userDao.save(user);
System.out.println(save);
}
}
批量添加文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void batchAddDocument() {
List<User> list = new ArrayList<>();
for (int i = 0; i < 10; i++) {
String name = "whn" + i;
String id = UUID.nameUUIDFromBytes(name.getBytes()).toString().replace("-", "");
User user = User.builder()
.id(id).address("南京鼓楼")
.age(18).email("147258369@qq.com")
.tel("1234567890")
.birthday(new Date())
.name(name).build();
list.add(user);
}
Iterable<User> users = userDao.saveAll(list);
users.forEach(System.out::println);
}
}
根据ID文档更新
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void updateDocument() {
String name = "whn1";
String id = "whn123456879";
User user = User.builder()
.id(id).address("徐州鼓楼")
.age(188).email("147258369@qq.com")
.tel("1234567890")
.birthday(new Date())
.name(name).build();
User save = userDao.save(user);
System.out.println(save);
}
}
条件更新文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void updateQueryDocument() {
String name = "whn";
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(QueryBuilders.matchQuery("name", name)).build();
UpdateQuery updateQuery = UpdateQuery.builder(searchQuery).withScript("ctx._source.name = '13546'").withScriptName("123456").withScriptType(ScriptType.STORED).build();
ByQueryResponse byQueryResponse = elasticsearchRestTemplate.updateByQuery(updateQuery, IndexCoordinates.of(User.class.getDeclaredAnnotation(Document.class).indexName()));
System.out.println(byQueryResponse.getTotal());
AllSearchDocument();
}
}
删除全部文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void DeleteAll() {
userDao.deleteAll();
}
}
根据id删除文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void DeleteDocumentById() {
String id = "whn12345116";
List<String> list = Arrays.asList(id);
userDao.deleteAllById(list);
}
}
条件删除文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void ConditionsDeleteDocument() {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name", "whn"));
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("age");
rangeQueryBuilder.lte(20);
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder()
.withQuery(boolQueryBuilder).withQuery(rangeQueryBuilder);
NativeSearchQuery nativeSearchQuery = nativeSearchQueryBuilder.build();
ByQueryResponse delete = elasticsearchRestTemplate.delete(nativeSearchQuery, User.class);
System.out.println(delete.getTotal());
}
}
检索全部文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void AllSearchDocument() {
Iterable<User> all = userDao.findAll();
all.forEach(System.out::println);
}
}
条件检索文档
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void advancedSearchDocument() {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name", "whn"));
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("age");
rangeQueryBuilder.lte(20);
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder()
.withQuery(boolQueryBuilder).withQuery(rangeQueryBuilder);
NativeSearchQuery nativeSearchQuery = nativeSearchQueryBuilder.build();
SearchHits<User> searchHits = elasticsearchRestTemplate.search(nativeSearchQuery, User.class);
searchHits.forEach(personSearchHit -> {
System.out.println(personSearchHit.getContent());
});
}
}
高亮检索
@SpringBootTest
@RunWith(SpringRunner.class)
class DocumentTest {
@Autowired
private UserDao userDao;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
@Test
void highLightSearchDocument() {
//高亮字段设置
HighlightBuilder.Field field = new HighlightBuilder
//高连字段名
.Field("name")
//高亮标签设置
.preTags("<span class=\"highlight\">")
.postTags("</span>");
// 匹配查询
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.matchQuery("name", "whn"));
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("age");
rangeQueryBuilder.lte(20);
// 构建查询对象
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder()
.withQuery(boolQueryBuilder).withQuery(rangeQueryBuilder).withHighlightFields(field);
NativeSearchQuery nativeSearchQuery = nativeSearchQueryBuilder.build();
//检索
SearchHits<User> searchHits = elasticsearchRestTemplate.search(nativeSearchQuery, User.class);
List<User> userList = new ArrayList<>();
// 遍历检索对象处理高亮内容
searchHits.forEach(personSearchHit -> {
User content = personSearchHit.getContent();
// 处理高亮
Map<String, List<String>> highlightFields = personSearchHit.getHighlightFields();
for (Map.Entry<String, List<String>> stringHighlightFieldEntry : highlightFields.entrySet()) {
String key = stringHighlightFieldEntry.getKey();
if (StringUtils.equals(key, "name")) {
List<String> fragments = stringHighlightFieldEntry.getValue();
StringBuilder sb = new StringBuilder();
for (String fragment : fragments) {
sb.append(fragment);
}
content.setName(sb.toString());
}
if (StringUtils.equals(key, "email")) {
List<String> fragments = stringHighlightFieldEntry.getValue();
StringBuilder sb = new StringBuilder();
for (String fragment : fragments) {
sb.append(fragment);
}
content.setEmail(sb.toString());
}
}
userList.add(content);
});
System.out.println(userList);
}
}
ES 使用sql进行查询
以下是一个使用 Elasticsearch SQL 查询的简单示例,假设我们有一个名为 employee
的索引,其中包含如下结构的数据:
{
"id": "1",
"first_name": "John",
"last_name": "Doe",
"age": 30,
"department": "Sales",
"salary": 50000
}
现在,我们可以使用 SQL 查询语句来检索和分析这些数据:
-- 基本查询
SELECT first_name, last_name, age FROM employee WHERE department = 'Sales';
-- 排序查询
SELECT * FROM employee ORDER BY salary DESC LIMIT 10;
-- 聚合查询
SELECT department, AVG(salary) as avg_salary FROM employee GROUP BY department;
要实际执行这些查询,你可以通过 Elasticsearch 的 REST API 或者使用支持 JDBC 的工具(如 SQL 客户端、BI 工具等)连接到 Elasticsearch。以下是如何通过 REST API 执行查询的一个示例:
curl -X POST -H 'Content-Type: application/json' -d '
{
"query": "SELECT first_name, last_name, age FROM employee WHERE department = 'Sales'"
}' 'http://localhost:9200/_sql'
请注意,上述示例中的 URL http://localhost:9200 应替换为你实际的 Elasticsearch 服务器地址。响应将是一个 JSON 对象,包含查询结果。
以上就是使用 Elasticsearch SQL 进行查询的简单示例。根据你的具体需求,可以编写更复杂的 SQL 查询来利用 Elasticsearch 的强大搜索和分析能力。
public static void translateConditions(String sql, RestHighLevelClient client) throws IOException {
Map<String, Object> map = new HashMap<>(2);
map.put("query", sql);
Request request = new Request("POST", "/_sql");
Gson gson = new Gson();
String jsonEntity = gson.toJson(map);
LOGGER.info("jsonEntity:" + jsonEntity);
request.setJsonEntity(gson.toJson(map));
Response response = client.getLowLevelClient().performRequest(request);
if (response.getStatusLine().getStatusCode() != 200) {
throw new RuntimeException("组合查询sql解析异常!");
}
// ...response 响应结果处理
}
SQL 转 DSL
public static QueryBuilder translateConditions(String sql, RestHighLevelClient client) throws IOException {
Map<String, Object> map = new HashMap<>(2);
map.put("query", sql);
Request request = new Request("POST", "/_sql/translate");
Gson gson = new Gson();
String jsonEntity = gson.toJson(map);
LOGGER.info("jsonEntity:" + jsonEntity);
request.setJsonEntity(gson.toJson(map));
Response response = client.getLowLevelClient().performRequest(request);
if (response.getStatusLine().getStatusCode() != 200) {
throw new RuntimeException("组合查询sql解析异常!");
}
JsonObject jsonObject = gson.fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class);
String queryJson = jsonObject.get("query").toString();
LOGGER.info("translated DSL is:" + queryJson);
return QueryBuilders.wrapperQuery(queryJson);
}