一、docker安装ELK
1.安装下载docker,从docker官网下载,windows和mac版都可以
链接https://www.docker.com/products/docker-desktop
2.启动docker桌面版
docker pull elasticsearch:7.5.0
docker pull logstash:7.5.0
docker pull kibana:7.5.0
版本最好统一,不然会有兼容类问题
等待镜像下载完成,docker images 可查看下载的镜像
二、启动elasticsearch和kibana
1、启动es
docker run -d -e ES_JAVA_POTS="-Xms256m -Xmx256m" -e "discovery.type=single-node" -p 9200:9200 -p 9300:9300 --name elasticsearch elasticsearch:7.5.0
这里指定了启动时的最大和最小内存,指定es的9200和9300端口,name为elasticsearch
启动完成后,在浏览器输入http://localhost:9200/ 可看到
代表es启动成功
2、启动kibana
docker run -d -p 5601:5601 --name kibana --link elasticsearch -e "ELASTICSEARCH_URL=http://127.0.0.1:9200" kibana:7.5.0
在浏览器输入localhost:5601可能会出现
Kibana server is not ready yet
说明还没有完全启动成功,等待片刻刷新页面就启动完成
3.可进入kibana上的DEV Tools 进行es Query 语句的书写
链接https://www.elastic.co/guide/cn/elasticsearch/guide/current/index.html
es的中文官方指南,只支持到了2.X版本,之后都只支持英文版本
三、基于RestHighLevelClient的Elasticsearchza在java中使用
1.pom包引入
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.6.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>demo</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>7.5.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.5.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.5.0</version>
</dependency>
<!--日志-->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.11.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
pom包,记得一定要引入rest-high-level-clien和rest-client包,不然生成es的client会报错
2.创建es-Client
package com.example.demo.esUtil;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
/**
* esClient
* @author devin
* @date 2020/4/10 20:39
*/
public class EsClient {
private static int port = 9200;
private static String host = "127.0.0.1"; //可以写 ip 但是 logstash 配置要修改
public static RestHighLevelClient getClient() {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost(host, port, "http")
// 可以指定集群的多个节点地址,将客户端负载均衡地向这个节点地址集发请求
));
return client;
}
}
3.通过java代码创建索引index
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
/**
* 创建索引
* @author devin
* @date 2020/4/10 20:45
*/
public class CreateIndexDemo {
private static Logger logger = LogManager.getRootLogger();
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
// 创建索引
// CreateIndexRequest request = new CreateIndexRequest("demo_index");
CreateIndexRequest request = new CreateIndexRequest("cars");
// 索引setting配置
request.settings(Settings.builder().put("index.number_of_shards", 5)
.put("index.number_of_replicas", 2) // 副本数
.put("analysis.analyzer.default.tokenizer", "standard")
);
// request.mapping("{" +
// "\"properties\": {" +
// "\"user\": {" +
// "\"type\": \"text\"" +
// "}," +
// "\"postDate\": {" +
// "\"type\": \"date\"" +
// "}," +
// "\"message\": {" +
// "\"type\": \"text\"" +
// "}" +
// "}" +
// "}", XContentType.JSON);
CreateIndexResponse response = null;
try {
response = client.indices().create(request, RequestOptions.DEFAULT);
if (response != null) {
boolean acknowledged = response.isAcknowledged();
boolean shardsAcknowledged = response.isShardsAcknowledged();
System.out.println("请求结果---------------");
System.out.println("acknowledged:" + acknowledged);
System.out.println("shardsAcknowledged:" + shardsAcknowledged);
}
} catch (IOException e) {
logger.error("创建index失败", e);
} finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
}
}
mapping可以根据需求自己制定需要的字段格式类型text,integer,date等,如果不指定mapping,那么在执行插入文档的时候会自动创建mapping和properties
4.新增单个文档
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
/**
* 新增文档
* @author devin
* @date 2020/4/11 10:54
*/
public class IndexDocumentDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "demo_index";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
// 创建索引
IndexRequest request = new IndexRequest(INDEX).id("1");
// 1.String方式添加数据
String jsonString = "{" +
"\"user\":\"devin\"," +
"\"postDate\":\"2020-04-01\"," +
"\"message\":\"today is a lucky day\"" +
"}";
request.source(jsonString, XContentType.JSON);
/* 2.通过map设置数据
Map<String, Object> jsonMap = new HashMap<>();
jsonMap.put("user", "devin");
jsonMap.put("postDate", "2020-04-01");
jsonMap.put("message", "today is a lucky day");
request.source(jsonMap);
*/
/* 3.通过Builder设置数据
try {
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
xContentBuilder.startObject();
xContentBuilder.field("user", "devin");
xContentBuilder.field("postDate", "2020-04-01");
xContentBuilder.field("message", "today is a lucky day");
xContentBuilder.endObject();
request.source(xContentBuilder);
} catch (IOException e) {
e.printStackTrace();
}
*/
/* 4.直接使用key-value
request.source(
"user","devin",
"postDate", "2020-04-01",
"message","today is a lucky day"
);
*/
/* 其他的一些可选设置
request.routing("routing"); //设置routing值
request.timeout(TimeValue.timeValueSeconds(1)); //设置主分片等待时长
request.setRefreshPolicy("wait_for"); //设置重刷新策略
request.version(2); //设置版本号
request.opType(DocWriteRequest.OpType.CREATE); //操作类别
*/
IndexResponse indexResponse = null;
try {
indexResponse = client.index(request, RequestOptions.DEFAULT);
} catch (ElasticsearchException e) {
if (e.status().equals(RestStatus.CONFLICT)) {
logger.error("文档已存在冲突");
}
logger.error("创建文档异常", e);
}catch (IOException e){
logger.error(e);
}finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
// 处理响应
if(indexResponse != null) {
String index = indexResponse.getIndex();
String id = indexResponse.getId();
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
System.out.println("新增文档成功!");
} else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
System.out.println("修改文档成功!");
}
System.out.println("index:" + index);
System.out.println("id:" + id);
}
}
}
新增doc有多种形式,jsonString类型,map类型,Builder类型,直接key-value类型;
5.批量新增文档
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import java.io.IOException;
/**
* 批量新增文档
*
* @author devin
* @date 2020/4/11 15:00
*/
public class BulkDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "demo_index";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
BulkRequest request = new BulkRequest(INDEX);
request.add(new IndexRequest(INDEX).id("1")
.source("user", "jack")
);
request.add(new IndexRequest(INDEX).id("2")
.source("postDate", "2020-04-02")
);
request.add(new IndexRequest(INDEX).id("3")
.source("message", "today is a happy day")
);
BulkResponse response = null;
try {
response = client.bulk(request, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error(e);
} finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
if (response != null) {
for (BulkItemResponse bulkItemResponse : response) {
DocWriteResponse docWriteResponse = bulkItemResponse.getResponse();
if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE
|| bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX) {
// 多次执行也只是匹配到的type为新增,其实操作为update
IndexResponse indexResponse = (IndexResponse) docWriteResponse;
logger.info("新增成功id:{}", indexResponse.getId());
} else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) {
UpdateResponse updateResponse = (UpdateResponse) docWriteResponse;
logger.info("更新成功id:{}", updateResponse.getId());
} else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) {
DeleteResponse deleteResponse = (DeleteResponse) docWriteResponse;
logger.info("删除成功id{}", deleteResponse.getId());
}
}
}
}
}
6.根据文档id获取文档doc
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
/**
* 获取文档
* @author devin
* @date 2020/4/11 14:45
*/
public class GetDocumentDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "demo_index";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
GetRequest request = new GetRequest(INDEX).id("1");
GetResponse response = null;
try {
response = client.get(request, RequestOptions.DEFAULT);
} catch (ElasticsearchException e) {
if (e.status() == RestStatus.NOT_FOUND) {
logger.error("没有找到文档");
}
} catch (IOException e) {
logger.error("获取文档异常", e);
}finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
if (response != null) {
String index = response.getIndex();
String id = response.getId();
if (response.isExists()) {
String sourceAsString = response.getSourceAsString();
logger.info("index:" + index + " id:" + id);
logger.info(sourceAsString);
}
}
}
}
7.搜索语句获取文档
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.ScoreSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import java.io.IOException;
import java.util.Arrays;
/**
* 搜索文档
* @author devin
* @date 2020/4/11 15:58
*/
public class SearchDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "demo_index";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
SearchRequest request = new SearchRequest(INDEX);
// 其实这些最后都会转化为String类型的 es query查询语句,只不过java封装了可以省却了写原生query语句
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 全匹配查询
/*
MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
searchSourceBuilder.query(matchAllQueryBuilder);
*/
// 精确查询
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("user", "devin");
searchSourceBuilder.query(termQueryBuilder);
searchSourceBuilder.from(0);
searchSourceBuilder.size(10);
//指定排序
/*
searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
searchSourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC));
*/
request.source(searchSourceBuilder);
SearchResponse response = null;
try {
response = client.search(request, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error(e);
}finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
if(response != null){
SearchHits hits = response.getHits();
SearchHit[] searchHits = hits.getHits();
Arrays.stream(searchHits).forEach( h ->
logger.info(h.toString())
);
}
}
}
8.聚合查询aggs
此聚合查询使用的是官方文档中的cars index,附带批量插入cars代码
官方文档链接:
https://www.elastic.co/guide/cn/elasticsearch/guide/current/_aggregation_test_drive.html
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
/**
* 批量新增文档 根据官方文档数据
* https://www.elastic.co/guide/cn/elasticsearch/guide/current/_aggregation_test_drive.html
* @author devin
* @date 2020/4/11 15:00
*/
public class CarsBulkDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "cars";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
BulkRequest request = new BulkRequest(INDEX);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 10000, \"color\" : \"red\", \"make\" : \"honda\", \"sold\" : \"2014-10-28\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 20000, \"color\" : \"red\", \"make\" : \"honda\", \"sold\" : \"2014-11-05\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 30000, \"color\" : \"green\", \"make\" : \"ford\", \"sold\" : \"2014-05-18\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 15000, \"color\" : \"blue\", \"make\" : \"toyota\", \"sold\" : \"2014-07-02\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 12000, \"color\" : \"green\", \"make\" : \"toyota\", \"sold\" : \"2014-08-19\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 20000, \"color\" : \"red\", \"make\" : \"honda\", \"sold\" : \"2014-11-05\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 80000, \"color\" : \"red\", \"make\" : \"bmw\", \"sold\" : \"2014-01-01\" }", XContentType.JSON
)
);
request.add(new IndexRequest(INDEX)
.source("{ \"price\" : 25000, \"color\" : \"blue\", \"make\" : \"ford\", \"sold\" : \"2014-02-12\" }", XContentType.JSON
)
);
BulkResponse response = null;
try {
response = client.bulk(request, RequestOptions.DEFAULT);
} catch (
IOException e) {
logger.error(e);
} finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
if (response != null) {
for (BulkItemResponse bulkItemResponse : response) {
DocWriteResponse docWriteResponse = bulkItemResponse.getResponse();
if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE
|| bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX) {
// 多次执行也只是匹配到的type为新增,其实操作为update
IndexResponse indexResponse = (IndexResponse) docWriteResponse;
logger.info("新增成功id:{}", indexResponse.getId());
} else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) {
UpdateResponse updateResponse = (UpdateResponse) docWriteResponse;
logger.info("更新成功id:{}", updateResponse.getId());
} else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) {
DeleteResponse deleteResponse = (DeleteResponse) docWriteResponse;
logger.info("删除成功id{}", deleteResponse.getId());
}
}
}
}
}
聚合查询
package com.example.demo.esUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.Avg;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.aggregations.metrics.Min;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
/**
* 聚合查询 已官方文档中的cars为例
* https://www.elastic.co/guide/cn/elasticsearch/guide/current/_one_final_modification.html
*
* @author devin
* @date 2020/4/11 16:17
*/
public class AggregationDemo {
private static Logger logger = LogManager.getRootLogger();
private static final String INDEX = "cars";
public static void main(String[] args) {
RestHighLevelClient client = EsClient.getClient();
// 1、创建search请求
SearchRequest request = new SearchRequest(INDEX);
// 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法,构造各种查询的方法都在这。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
//加入聚合
//字段值项分组聚合
TermsAggregationBuilder aggregation = AggregationBuilders.terms("colors")
.field("color.keyword");
// 计算通过color分组后的平均价格
aggregation.subAggregation(
AggregationBuilders.avg("avg_price").field("price")
);
aggregation.subAggregation(
AggregationBuilders.terms("make").field("make.keyword")
.subAggregation(AggregationBuilders.max("max_price").field("price"))
.subAggregation(AggregationBuilders.min("min_price").field("price"))
);
searchSourceBuilder.aggregation(aggregation);
System.out.println("----------");
System.out.println(searchSourceBuilder.toString());
request.source(searchSourceBuilder);
//3、发送请求
SearchResponse response = null;
try {
response = client.search(request, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error(e);
} finally {
try {
client.close();
} catch (IOException e) {
logger.error(e);
}
}
if (response != null && response.status() == RestStatus.OK) {
Aggregations aggregations = response.getAggregations();
Terms colors = aggregations.get("colors");
for (Terms.Bucket bucket : colors.getBuckets()) {
logger.info("------------------");
logger.info("color:{}", bucket.getKeyAsString());
Aggregations aggregations1 = bucket.getAggregations();
Avg avg_price = aggregations1.get("avg_price");
logger.info("avg_price:{}", avg_price.getValue());
Terms make = bucket.getAggregations().get("make");
for (Terms.Bucket bucket2 : make.getBuckets()) {
logger.info("make:{}", bucket2.getKeyAsString());
Max max_price = bucket2.getAggregations().get("max_price");
Min min_price = bucket2.getAggregations().get("min_price");
logger.info("max_price=" + max_price.getValue());
logger.info("min_price=" + min_price.getValue());
}
}
}
}
}