ElasticSearch配置备忘录

ES大数据量查询并同步至GBase

问题一：不变更SpringBoot版本且与ES版本号对应

问题二：RestHighLevelClient使用trackTotalHits(true)解除10000条的上限但是无效（定义游标）

问题三：GBase批量数据入库导致行锁（GBase是高可用、高性能的但却不是高并发的）

1.Maven

<parent>
	<groupId>org.springframework.boot</groupId>
	<artifactId>spring-boot-starter-parent</artifactId>
	<version>2.5.6</version>
	<relativePath/> <!-- lookup parent from repository -->
</parent>

<!-- SpringBoot2.5.6对应Elasticsearch版本依赖  -->
<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>6.8.6</version>
</dependency>
<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>6.8.6</version>
</dependency>

2.代码样例

public class EsHitsAsyncDemo{
	// 指定检索条件
    SearchRequest request = new SearchRequest().indices("ES索引名称");
    // 配置游标失效时间
    request.scroll(TimeValue.timeValueHours(2));
    SearchSourceBuilder builder = new SearchSourceBuilder();
    builder.size(10000);
    builder.query(QueryBuilders.matchAllQuery());
    request.source(builder);

    // 发送请求到ES
    SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
    // 定义游标
    String scrollId = null;
    if (response != null && response.getHits().getHits().length > 0) {
	    // 处理响应结果
	    for (SearchHit searchHits : response.getHits().getHits()) {
		    log.info("模拟数据样例:{}", searchHits);
		    EsDemoDO esDemoDO = JsonUtils.string2Obj(searchHits.getSourceAsString(), EsDemoDO .class);
	        this.asyncData(esDemoDO);
	    }
     	scrollId = response.getScrollId();
     }
     // 循环查询直到没有游标返回
     while (true){
		if(scrollId == null){
			break;
        }
        SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueHours(2));;
        response = restHighLevelClient.scroll(searchScrollRequest, RequestOptions.DEFAULT);
        if (response != null && response.getHits().getHits().length > 0) {
        	for (SearchHit searchHits : response.getHits().getHits()) {
            	log.info("模拟数据样例:{}", searchHits);
		    EsDemoDO esDemoDO = JsonUtils.string2Obj(searchHits.getSourceAsString(), EsDemoDO .class);
	        this.asyncData(esDemoDO);
            }
            scrollId = response.getScrollId();
		}else {
            break;
        }
    }
    // 关闭ES客户端对象
    restHighLevelClient.close();
}

private void asyncData(EsDemoDO esDemoDO) throws SQLException {
    log.info("ES数据入库：{}", esDemoDO);
    //获取数据库DataSource
    DataSource dataSource = (DataSource) SpringContextUtils.getBean("****");
    Connection conn = dataSource.getConnection();
    //自定义入库SQL
    String sql = "";
    PreparedStatement pst = conn.prepareStatement(sql);
    pst.setString(1, esDemoDO.getProvincecode());
    java.sql.Timestamp insertTime = new java.sql.Timestamp(DateTimeUtils.parseDateTime(esDemoDO.getInserttimeforhis()).getTime());
    pst.setTimestamp(2, insertTime);
    pst.execute();
    pst.close();
    conn.close();
}

ES与SpringBoot普通整合【采用ElasticsearchRestTemplate】

1.Maven

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>

2.代码样例

/**
 * @author lzh
 * @date 2022/6/10 14:51
 */
@Data
@Document(indexName = "demo")
public class Demo {
	@Id
    private String demoId;
    
    @Field(type = FieldType.Keyword)
    private String demoName;

	@Field(type = FieldType.Text)
    private String remark;
	
    @Field(type = FieldType.Date)
    private Date Datetime;
}

public class EsDemo {
	@Autowired
    private ElasticsearchRestTemplate esRestTemplate;
	
	/**
	 * ES条件查询整合笔记
	 */
	public SearchHits<Demo> queryByWhere(String demoId, String year, List<String> demoName) {
		NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
		builder.withTrackTotalHits(true);
        // 构建boolQuery查询
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        // 主键查询
        if (StringUtils.hasText(demoId)) {
            boolQueryBuilder.must(QueryBuilders.idsQuery().addIds(demoId));
        } 
        // 区间查询
        if (StringUtils.hasText(year)) {
        	String nextYear = String.valueOf(Integer.parseInt(year) + 1);
            boolQueryBuilder.filter(QueryBuilders.rangeQuery("Datetime").gte(year).format("yyyy").lt(nextYear).format("yyyy"));
        }
        // 类似与SQL中的[in('a','b','c')]
        if (!CollectionUtils.isEmpty(demoName)) {
            boolQueryBuilder.must(QueryBuilders.termsQuery("demo_name", demoName));
        }
        // 精确查询某字段
        boolQueryBuilder.filter(QueryBuilders.matchQuery("country", "CN"));
        // 按照时间排序
        FieldSortBuilder dateTimeSort = SortBuilders.fieldSort("Datetime").order(SortOrder.ASC);
		NativeSearchQuery build = builder.withQuery(boolQueryBuilder)
                						 .withSort(dateTimeSort)
                						 .build();
        // 返回结果自行根据需求使用Lambda组装数据结构        						 
        return esRestTemplate.search(build, Demo.class);  						 
	}
}

除了查询别的什么的不值一提一看就会