简介:ElasticSearch:基于Apache Lucene的开源搜索引擎,面向文档的NoSQL数据库
- 分布式的实时文件存储,每个字段都被索引并可被搜索
- 分布式的实时分析搜索引擎
- 可以扩展到上百台服务器,处理PB级结构化或非结构化数据
Springboot-ES
pom:
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
</dependency>
yml: #文件中配置jest注解@Autowired注入就行
spring:
elasticsearch:
jest:
uris: http://localhost:9200/
read-timeout: 6000
ES工具
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.core.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
@Slf4j
public class ElasticSearchFactory {
public static Map<String,List<ESObject>> map = new HashMap<String,List<ESObject>>();
/**
* 创建Search对象
* @Author Guanxiangfu
* @Date 2017/12/1 14:23
*/
public static Search getSearch(String string, String index, String type) {
return new Search.Builder(string)
.addIndex(index)
.addType(type)
.build();
}
/**
* 创建Get对象
* @Author Guanxiangfu
* @Date 2017/12/1 14:23
*/
public static Get getGet(String id, String index, String type) {
return new Get.Builder(index, id)
.type(type)
.build();
}
/**
* 获取Index对象
* @Author Guanxiangfu
* @Date 2017/12/8 11:14
*/
public static Index getIndex(Object object, String index, String type, String id) {
return new Index.Builder(object)
.id(id)
.index(index)
.type(type)
.build();
}
/**
* 获取删除对象
* @Author Guanxiangfu
* @Date 2017/12/8 14:51
*/
public static Delete getDelete(String id, String index, String type) {
return new Delete.Builder(id)
.index(index)
.type(type)
.build();
}
/**
* 创建修改对象
* @Author Guanxiangfu
* @Date 2017/12/11 10:19
*/
public static Update getUpdate(Map<String, Object> object, String index, String type, String id) {
Map<?, ?> data = ArrayUtils.toMap(new Object[][]{{"doc", object}});
return new Update.Builder(data)
.index(index)
.type(type)
.id(id)
.build();
}
/**
* 创建连接
* @Author Guanxiangfu
* @Date 2017/12/25 13:51
*/
public static JestClient getJestClient(String url) {
HttpClientConfig clientConfig = new HttpClientConfig.Builder(url)
.readTimeout(18000)//设置默认连接时间为18s
.multiThreaded(true).build();
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(clientConfig);
return factory.getObject();
}
/**
* 批量插入
*/
public static Bulk getBulk(String index, String type, List<Map> data) {
Bulk.Builder builder = new Bulk.Builder().defaultIndex(index).defaultType(type);
for (Map datum : data) {
String id = (String) datum.get("ESID");
builder.addAction(new Index.Builder(datum).id(id).build());
}
return builder.build();
}
/**
* 删除索引
* @Author Guanxiangfu
* @Date 2018/3/27 21:05
*/
public static DeleteByQuery getDeleteByQuery(String index, String type){
return new DeleteByQuery.Builder("{\"query\":{\"match_all\":{}}}")
.addIndex(index)
.addType(type)
.build();
}
/**
* 获取Mapping结构
* @Author Guanxiangfu
* @Date 2018/3/27 21:13
*/
public static GetMapping getMapping(String index){
return new GetMapping.Builder()
.addIndex(index)
.build();//获取到数据可用KeySet获取type名
}
/**
* 获取Count对象
* @Author Guanxiangfu
* @Date 2018/3/27 21:16
*/
public static Count getCount(String index, String type){
return new Count.Builder()
.addIndex(index)
.addType(type)
.build();//获取到数据可用getCount()获取数据量
}
/**
* 创建Mapping
* @Author GuanXiangfu
* @return boolean
* @throws BusinessException
*/
@Override
public boolean createMapping(String index, String dataSet, String tableName) throws BusinessException {
//获取mapping
String string = getMapping(index, dataSet, tableName);
PutMapping.Builder builder = new PutMapping.Builder(index, tableName, string);
try {
JestResult result = jestClient.execute(builder.build());
if(result == null || !result.isSucceeded()){
throw new RuntimeException(result.getErrorMessage());
}
} catch (Exception e) {
log.error("创建错误",e);
String reason = JSONObject.parseObject(e.getMessage()).getString("reason");
res.add("表名:" + tableName + " 插入失败! 原因:" + reason);
return false;
}
return true;
}
}
ES语法
创建mapping
{“mappings”: {“process”: {“properties”: {“shape”: {“type”: “geo_shape”},”id”: {“type”: “keyword”},”name”: {“type”: “keyword”},”opacity”: {“type”: “keyword”}}}}}字段查询
可以添加index、或type名,也可以不填,效果不同
{“query”: {“terms”: {“status”: [304,302]}}} terms可以查多个范围查询
gt :: 大于
gte:: 大于等于
lt :: 小于
lte:: 小于等于
可以添加index、或type名,也可以不填,效果不同返回特点字段查询
{“_source”:{“include”:”DSID”},”query”:{“match_all”:{}},”size”:3000}模糊查询
{“query”:{“wildcard”:{“dir_id”:”oo“}},”size”:200}
{“from”:0,”query”:{“bool”:{“minimum_should_match”:1,”must”:[{“match”:{“XB”:”2 “}},{“wildcard”:{“XM”:”王*”}}],”should”:[{“range”:{“CSRQ”:{“gte”:”2010-09-01”,”lte”:”2014-09-01”}}}]}},”size”:10}
ES索引操作
- -x 指定http请求的方法
- -d 指定要传输的数据
- – HEAD GET POST PUT DELETE
- curl后添加-i 参数,就能得到反馈头文件
index新增操作:当ES数据没有时,进行添加操作,有数据时,覆盖操作
update修改操作:没有数据时,不做添加,有数据时,改重复的字段,其他字段保留不动
upsert新增\修改:有数据时进行修改,没数据时新增
curl –XPOST ‘http://localhost:9200/index/type/id/_update’ _d’
{“doc”:{“age”:”18”,”name”:”gxf”},”upsert”:{“id”:”18”,”name”:”gxf”}}’
delete删除:删除索引的type,删除索引
curl -XPOST “localhost:9200/epdmv1/_delete_by_query?pretty” -H ‘Content-Type: application/json’ -d’{“query”:{“match_all”:{}}}’
create索引:
curl -XPOST ‘http://localhost:9200/bjsxt’
create类型:
curl -XPOST http://localhost:9200/bjsxt/employee/1 -d
‘{“first_name” : “John”,”last_name” : “Smith”,”age” : 25,”about” : “I love to go rock climbing”,
“interests”: [ “sports”, “music” ]}’
curl -XPUT http://localhost:9200/bjsxt/emp/2?op_type=create -d ‘{“name”:“zs”,”age”:25}’
curl -XPUT http://localhost:9200/bjsxt/emp/2/_create -d ‘{“name”:“laoxiao”,”age”:25}’
get查询ID:
检索文档中的一部分,只显示name,age字段
curl -XGET http://localhost:9200/bjsxt/employee/1?_source=name,age
如果只需要source的数据
curl -XGET http://localhost:9200/bjsxt/employee/1/_source
查询所有
curl -XGET http://localhost:9200/bjsxt/employee/_search
根据条件进行查询
curl -XGET http://localhost:9200/bjsxt/employee/_search?q=last_name:Smith
update部分更新:
curl -XPOST http://localhost:9200/bjsxt/employee/1/_update?version=3 -d ‘{“doc”:{“city”:”beijing”,”car”:”BMW”}}’
检查文档是否存在:你可以使用HEAD来替代GET方法,这样就只会返回HTTP头文件:
curl -i -XHEAD http://localhost:9200/bjsxt/employee/1
Elasticsearch的版本控制:
首先得到需要修改的文档,获取版本(_version)号
curl -XGET http://localhost:9200/bjsxt/employee/1
在执行更新操作的时候把版本号传过去
curl -XPUT http://localhost:9200/bjsxt/employee/1?version=2 -d ‘{“name”:”zs”,”age”:25}’
ElasticSearch查询数量:
http://localhost:9200/index/_count
健康值查询:
http://localhost:9200/_cluster/health?pretty
时间范围查询:
POST log_etl_201808/_search
{
"query": {
"constant_score" : {
"filter" : {
"range" : {
"startTime" : {
"gt" : "2018-08-25 8:30:14",
"lt" : "2018-08-25 12:30:14"
}
}
}
}
}
}
多条件值查询
POST log_etl_201808/_search
{
"query": {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{ "term" : {"processId": "KAFKA" } },
{ "term" : { "dataRegion": "JD" } }
]
}
}
}
}
}
组合查询(时间、多条件)
POST log_etl_201808/_search
{
"query": {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{ "term" : {"processId": "KAFKA" } },
{ "term" : { "dataRegion": "JD" } } ,
{
"range" : {
"startTime" : {
"gt" : "2018-08-25 8:30:14",
"lt" : "2018-08-25 12:30:14"
}
}
}
]
}
}
}
}
}