Elasticsearch基本使用

一、Kibana：可视化界面

1、创建一个名为tets_lsl的索引，设置索引的分片数为3，备份数为2。

PUT test_lsl
或
PUT test_lsl
{
    "settings" : {
        "index" : {
            "number_of_shards" : 3,
            "number_of_replicas" : 2
        }
    }
}
或
PUT test_lsl
{
    "settings" : {
        "number_of_shards" : 3,
        "number_of_replicas" : 2
    }
}

说明：
默认的分片数是5到1024
默认的备份数是1
索引的名称必须是小写的，不可重名

2、添加映射

 PUT /test_lsl/_mapping
 {
   "properties":{
     "name":{
       "type":"text"
     },
     "age":{
       "type":"integer"
     },
     "addr":{
       "type":"keyword"
     }
   }
 }

3、添加数据

指定id
POST /test_lsl/_doc/1
不指定id
POST /test_lsl/_doc
{
  "name":"小王",
  "age":18,
  "addr":"秦国"
}

4、查询数据

#查询所有
GET /test_lsl/_search
GET /test_lsl/_search
{
  "query": {
    "match_all": {}
  }
}
#根据条件查询
{
  "query": {
    "bool": {
      "must": [
        {
          "terms": {
          	#如果age是text类型，查询字段替换为为"age.keyword"
            "age": [
              "187",
              "13"
            ],
            "boost": 1
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  },
  "_source": {
  	#查询字段
    "includes": [
    ],
 	 #排除字段
    "excludes": []
  }
}
#

5、删除数据

#删除指定id数据
DELETE  /test_lsl/_doc/1
# 清空满足条件数据
POST person1/_delete_by_query
{  
  "query": {
    "match_all": {}
  }
}

6、跟新数据

跟新数据要添加所有数据（类似于先删除数据，再添加数据）

1、先查询对应数据
GET  /test_lsl/_search
{
  "query": {
    "match": {
      "recordId": "210391039470214144"
    }
  }
}

2、通过id更新文档
PUT /test_lsl/_doc/对应索引id
{
  "name":"小王",
  "age":18,
  "addr":"秦国"
}

7、分词器：ik_max_word（更细致）和ik_smart模式。

二、Elasticsearch在java中的使用

1、导入依赖

		<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>

2、创建配置类

@Configuration
public class ElasticSearchConfig {

    @Value("${elasticsearch.host}")
    private String esHost;

    @Value("${elasticsearch.port}")
    private Integer esPort;
    
    @Bean
    public RestHighLevelClient restHighLevelClient() {
        return new RestHighLevelClient(
            RestClient.builder(new HttpHost(esHost, esPort, null)));
    }
}

3、多条件查询BoolQueryBuilder

BoolQueryBuilder queryBuilder= QueryBuilders.boolQuery();
1.matchAllQuery匹配所有
queryBuilder.matchAllQuery();
//字段存在
QueryBuilders.existsQuery("organization")

2.termQuery精准匹配，大小写敏感且不支持
queryBuilder.termQuery("key", value) 一次匹配一个值，完全匹配
queryBuilder.termsQuery("key", obj1, obj2..)   一次匹配多个值
queryBuilder.termsQuery("key", List<T> list)   一次匹配多个值

3.matchPhraseQuery对中文精确匹配
queryBuilder.matchPhraseQuery("key", value)

4.matchQuery("key", Obj) 单个匹配
queryBuilder.matchQuery(key, value);

5.multiMatchQuery("text", "field1", "field2"..); 
queryBuilder.multiMatchQuery(value, key1, key2, key3);

6.组合查询
  注：must:and、mustNot:not 、should:or
 queryBuilder.must(QueryBuilders.termQuery("name", "小王"))
            .mustNot(QueryBuilders.termQuery("age", 18))
            .should(QueryBuilders.termQuery("addr", "秦国"));

4、构建查询条件（仅供参考）

#查询参数如果为text类型，后面拼接keyword，如："username"-->"username.keyword"

    /**
     * 构建单值查询条件
     *
     * @param indices       索引
     * @param paramName     参数名称
     * @param value         查询字段
     * @param queryFields   查询返回字段
     * @param excludeFields 排除字段
     */
    public static SearchRequest matchSearchRequest(String indices, String paramName, Object value,
        String[] queryFields,
        String[] excludeFields) {
        SearchRequest searchRequest = new SearchRequest(indices);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(paramName, value);
        sourceBuilder.query(matchQueryBuilder);
        sourceBuilder.fetchSource(queryFields, excludeFields);
        searchRequest.source(sourceBuilder);
        return searchRequest;
    }

	 /**
     * 构建多值查询条件
     *
     * @param indices       索引
     * @param paramName     参数名称
     * @param value         查询字段集合
     * @param queryFields   查询返回字段
     * @param excludeFields 排除字段
     */
    public static SearchRequest searchRequest(String indices, String paramName, List<?> value,
        String[] queryFields,
        String[] excludeFields) {
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
        boolQueryBuilder.must(QueryBuilders.termsQuery(paramName, value));
        sourceBuilder.query(boolQueryBuilder);
        sourceBuilder.fetchSource(queryFields, excludeFields);
        SearchRequest searchRequest = new SearchRequest(indices);
        searchRequest.source(sourceBuilder);
        return searchRequest;

    /**
     * 字段数组包含参数查询
     *
     * @param indices       索引
     * @param paramName     参数名称(存储值为数组)
     * @param value         查询字段
     * @param queryFields   查询返回字段
     * @param excludeFields 排除字段
     */
    public static SearchRequest searchRequestInArray(String indices, String paramName, Object value,
        String[] queryFields,
        String[] excludeFields) {
        SearchRequest searchRequest = new SearchRequest(indices);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(QueryBuilders.termsQuery(paramName, value));
        sourceBuilder.fetchSource(queryFields, excludeFields);
        searchRequest.source(sourceBuilder);
        return searchRequest;
    }
    }

    /**
     * 计算字段数组包含参数的值的和
     *
     * @param indices        索引
     * @param paramName      参数名称(存储值为数组)
     * @param value          查询字段
     * @param resultName     统计结果字段名
     * @param calculateField 被计算字段
     */
    public static SearchRequest sumSearchRequestInArray(String indices, String paramName, Object value,
        String resultName, String calculateField) {
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(QueryBuilders.termsQuery(paramName, value));
        sourceBuilder.fetchSource(false);
        sourceBuilder.aggregation(AggregationBuilders.sum(resultName).field(calculateField));
        SearchRequest searchRequest = new SearchRequest(indices);
        searchRequest.source(sourceBuilder);
        return searchRequest;
    }

4.1构建多条件查询

		BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
        boolQueryBuilder1.must(
            QueryBuilders.multiMatchQuery(key, value).type(MultiMatchQueryBuilder.Type.PHRASE)
        );
        BoolQueryBuilder boolQueryBuilder2 = QueryBuilders.boolQuery();
        boolQueryBuilder2.should(
            QueryBuilders.multiMatchQuery(key, value).type(MultiMatchQueryBuilder.Type.PHRASE)
        );
        BoolQueryBuilder boolQueryBuilder3 = QueryBuilders.boolQuery();
        boolQueryBuilder2.mustNot(
            QueryBuilders.multiMatchQuery(key, value).type(MultiMatchQueryBuilder.Type.PHRASE)
        );
        boolQuery.must(boolQueryBuilder1);
        boolQuery.must(boolQueryBuilder2);
        boolQuery.must(boolQueryBuilder3);

4.2取消分词器

条件构造后面添加：type(MultiMatchQueryBuilder.Type.PHRASE)
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must().type(MultiMatchQueryBuilder.Type.PHRASE));

5、封装查询结果

    /**
     * 封装返回参数到实体类
     *
     * @param search    elasticsearch查询结果
     * @param classType 返回类型
     * @param <T>       返回类型
     * @return T
     */
    public static <T> List<T> convertSearchResponseToEntity(SearchResponse search, Class<T> classType) {
        List<T> results = new ArrayList<>();
        for (SearchHit hit : search.getHits().getHits()) {
            Map<String, Object> map = hit.getSourceAsMap();
            T result = JSON
                .parseObject(JSON.toJSONString(map), classType);
            results.add(result);
        }
        return results;
    }

    /**
     * 封装基本类型返回参数
     *
     * @param search    elasticsearch查询结果
     * @param classType 返回类型
     * @param <T>       返回类型
     * @return T
     */
    public static <T> List<T> convertSearchResponse(SearchResponse search, Class<T> classType) {
        List<T> results = new ArrayList<>();
        for (SearchHit hit : search.getHits().getHits()) {
            Collection<Object> values = hit.getSourceAsMap().values();
            for (Object value : values) {
                T result = JSON
                    .parseObject(JSON.toJSONString(value), classType);
                results.add(result);
            }
        }
        return results;
    }

6、更新数据

    public void testUpdateEs(String indices) {
        UpdateByQueryRequest updateByQuery = new UpdateByQueryRequest("my_index");
        //设置分片并行
        updateByQuery.setSlices(2);
        //设置版本冲突时继续执行
        updateByQuery.setConflicts("proceed");
        //设置更新完成后刷新索引 ps很重要如果不加可能数据不会实时刷新
        updateByQuery.setRefresh(true);
        //查询条件如果是and关系使用must 如何是or关系使用should
        List<String> list = List.of("10", "20");
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
            .must(new MultiMatchQueryBuilder("age", "title", "username").type(MultiMatchQueryBuilder.Type.PHRASE));
        updateByQuery.setQuery(boolQueryBuilder);
        //设置要修改的内容可以多个值多个用；隔开
        1.替换 =
        updateByQuery.setScript(new Script("ctx._source['eventId'] =6L"));
        2.追加 +=
        updateByQuery.setScript(new Script("ctx._source['eventId'] +=6L"));
		3.字段为数组
		updateByQuery.setScript(new Script("ctx._source['age'].add(15L)"));
		4.字段为数组并加上判定if
        updateByQuery.setScript(
            new Script(
                "if(ctx._source['age'] == null) {ctx._source['age']=[];ctx._source['age'].add(15L)} else{ctx._source['age'].add(15L)} "));
        try {
            BulkByScrollResponse response = client.
                updateByQuery(updateByQuery, RequestOptions.DEFAULT);
            System.out.println();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

注：批量更新数据避免多次动态编译脚本，从而避免超过编译次数限制。
使用索引脚本或者带参数的脚本来替代您目前的内联脚本，这样可以减少脚本动态编译的次数
updateByQuery.setScript(new Script(ScriptType.INLINE, "painless", "ctx._source['recordId'] = params.recordId", Collections.singletonMap("recordId", id)));

7、查询处理es所有数据百万级

//引入依赖
	<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
		</dependency>

//构造查询参数
public static void  matchSearchRequest(
        String[] queryFields,
        String[] excludeFields,String indices) throws Exception {
        SearchRequest searchRequest = new SearchRequest(indices);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //设置每次查询数量
        sourceBuilder.size(1000);
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
            .must(QueryBuilders.matchAllQuery());
        sourceBuilder.query(boolQueryBuilder);
        sourceBuilder.fetchSource(queryFields, excludeFields);
        searchRequest.source(sourceBuilder);
        searchAllByScroll(searchRequest);
    }

    /**
     * 针对查询请求，通过 Scroll 查询所有，不分页
     * @param request 查询请求
     * @return
     */
    public static void searchAllByScroll(SearchRequest request) throws Exception {

        RestHighLevelClient client = new RestHighLevelClient(
            RestClient.builder(new HttpHost("192.168.210.110", 29200, null))
                .setHttpClientConfigCallback(requestConfig ->
                    requestConfig.setKeepAliveStrategy((response, context) -> TimeUnit.MINUTES.toMillis(3)))
                .setRequestConfigCallback(
                    requestConfigBuilder -> requestConfigBuilder
                        .setConnectTimeout(3000)
                        .setSocketTimeout(120 * 1000)));

        request.scroll(TimeValue.timeValueSeconds(10));

        SearchResponse response = client.search(request, RequestOptions.DEFAULT);
        String scrollId = response.getScrollId();
        SearchHit[] searchHits = response.getHits().getHits();

        while (searchHits != null && searchHits.length > 0) {
            for (SearchHit hit : searchHits) {
           		 //具体操作数据
                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            	MQProducer.send("192.168.210.51:39876", "resend_ic_tjsj-sz-info", sourceAsMap);
            }
            response = client.scroll(new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueSeconds(10)), RequestOptions.DEFAULT);
            scrollId = response.getScrollId();
            searchHits = response.getHits().getHits();
        }
        
		//清除滚动，否则影响下次查询
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollId);
        ClearScrollResponse clearScrollResponse = null;
        try {
            clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
        } catch (IOException e) {
            System.out.println("滚动查询删除失败");
        }
        //清除滚动是否成功
        boolean succeeded = clearScrollResponse.isSucceeded();
        }

​