es java api操作

 1，restHighLevelClient

<dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.8.13</version>
        </dependency>

        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client</artifactId>
            <version>6.8.13</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>6.8.13</version>
        </dependency>

2，注册bean

 
import java.util.ArrayList;
 
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import org.elasticsearch.client.RestClientBuilder.RequestConfigCallback;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
 
@Configuration
public class EsConfiguration {
 
	private static String hosts = "127.0.0.1"; // 集群地址，多个用,隔开
	private static int port = 9200; // 使用的端口号
	private static String schema = "http"; // 使用的协议
	private static ArrayList<HttpHost> hostList = null;
 
	private static int connectTimeOut = 1000; // 连接超时时间
	private static int socketTimeOut = 30000; // 连接超时时间
	private static int connectionRequestTimeOut = 500; // 获取连接的超时时间
 
	private static int maxConnectNum = 100; // 最大连接数
	private static int maxConnectPerRoute = 100; // 最大路由连接数
 
	static {
		hostList = new ArrayList<>();
		String[] hostStrs = hosts.split(",");
		for (String host : hostStrs) {
			hostList.add(new HttpHost(host, port, schema));
		}
	}
 
	@Bean
	public RestHighLevelClient client() {
		RestClientBuilder builder = RestClient.builder(hostList.toArray(new HttpHost[0]));
		// 异步httpclient连接延时配置
		builder.setRequestConfigCallback(new RequestConfigCallback() {
			@Override
			public Builder customizeRequestConfig(Builder requestConfigBuilder) {
				requestConfigBuilder.setConnectTimeout(connectTimeOut);
				requestConfigBuilder.setSocketTimeout(socketTimeOut);
				requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
				return requestConfigBuilder;
			}
		});
		// 异步httpclient连接数配置
		builder.setHttpClientConfigCallback(new HttpClientConfigCallback() {
			@Override
			public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
				httpClientBuilder.setMaxConnTotal(maxConnectNum);
				httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);
				return httpClientBuilder;
			}
		});
		RestHighLevelClient client = new RestHighLevelClient(builder);
		return client;
	}
 
}

3，bulk批处理：

    @Override
    public void asyncBulkUpdateEs(String index, String type, List<Map<String, Object>> mapList) {
        BulkRequest request = new BulkRequest();
        for (Map<String, Object> map : mapList) {
           //新增或者全量替换，慎用
            //request.add(new IndexRequest(index, type, map.get("id") == null ? null : map.get("id").toString()).source(map, XContentType.JSON));
//更新
request.add(new UpdateRequest(index, type, map.get("id") == null ? null : map.get("id").toString()).retryOnConflict(3).doc(map, XContentType.JSON));
        }
        //restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
        restHighLevelClient.bulkAsync(request, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() {
            @Override
            public void onResponse(BulkResponse bulkResponse) {
                if (bulkResponse.hasFailures()) {
                    log.error("es bulk 操作失败，参数:{}", JSONObject.toJSONString(mapList));
                }
            }

               @Override
            public void onFailure(Exception e) {
                log.error("es bulk 操作异常，参数:{}", JSONObject.toJSONString(mapList));
                log.error("异常信息:", e);
            }
        });


    }

   

4，EsBulkProcessor  批处理

@Configuration
@Slf4j
public class EsBulkProcessor {

    @Resource
    private RestHighLevelClient restHighLevelClient;

    @Bean(name = "bulkProcessor") // 可以封装为一个bean，非常方便其余地方来进行 写入 操作
    public BulkProcessor bulkProcessor() {

        BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
                (request, bulkListener) -> restHighLevelClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);

        return BulkProcessor.builder(bulkConsumer, new BulkProcessor.Listener() {
            @Override
            public void beforeBulk(long executionId, BulkRequest request) {
                // todo do something
                int i = request.numberOfActions();
                log.error("ES 同步数量{}", i);
            }
    @Override
            public void afterBulk(long executionId, BulkRequest request, BulkResponse bulkResponse) {
                // todo do something


                if (bulkResponse.hasFailures()) {
                    log.error("es bulkProcessor 操作失败，FailureMessage:{}", bulkResponse.buildFailureMessage());
                } else {
                    log.info("es bulkProcessor 操作成功");
                }

                /*Iterator<BulkItemResponse> iterator = bulkResponse.iterator();
                while (iterator.hasNext()) {
                    System.out.println(JSON.toJSONString(iterator.next()));
                }*/

            }

 @Override
            public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
                // todo do something
                log.error("写入ES 重新消费");
            }
        }).setBulkActions(1000) //  达到刷新的条数
                .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.MB)) // 达到 刷新的大小
                .setFlushInterval(TimeValue.timeValueSeconds(5)) // 固定刷新的时间频率
                .setConcurrentRequests(1) //并发线程数
                .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) // 重试补偿策略
                .build();

    }
}

具体使用

bulkProcessor.add(xxRequest)

bulkProcessor.add(new UpdateRequest(index,
                                    type, id).retryOnConflict(5).doc(JSON.toJSONString(updateMap), XContentType.JSON));