restHighLevelClient客户端连接elasticsearch实现数据迁移

elasticsearch数据迁移

    public void syncEsData(String index) throws IOException {
        if (StringUtils.isBlank(index)) {
            throw new RuntimeException("index name is present!");
        }
        SearchRequest searchRequest = new SearchRequest(index);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().sort("create_time", SortOrder.ASC).size(1000);
        searchSourceBuilder.query(QueryBuilders.rangeQuery("create_time").gt("2021-03-10 08:26:18"));
        searchRequest.source(searchSourceBuilder);
        Scroll scroll = new Scroll(TimeValue.timeValueHours(24));
        searchRequest.scroll(scroll);
        // restHighLevelClient是连接到数据源的客户端
        SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);

        String scrollId = response.getScrollId();

        SearchHit[] hits = response.getHits().getHits();

        List<String> sourceList = new ArrayList<>();
        for (SearchHit hit : hits) {
            sourceList.add(hit.getSourceAsString());
        }

        syncData(index, sourceList);

        while (hits != null && hits.length > 0) {
            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
            scrollRequest.scroll(scroll);
            try {
                response = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
            } catch (IOException e) {
                e.printStackTrace();
            }
            scrollId = response.getScrollId();
            hits = response.getHits().getHits();
            if (hits != null && hits.length > 0) {
                sourceList = new ArrayList<>();
                for (SearchHit hit : hits) {
                    sourceList.add(hit.getSourceAsString());
                }
                syncData(index, sourceList);
            }
        }

        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollId);
        ClearScrollResponse clearScrollResponse = null;
        clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
    }

    private RestHighLevelClient getNewClient() {
        // es连接地址
        RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost("127.0.0.1", 9200, "http"))
                .setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
                    @Override
                    public RequestConfig.Builder customizeRequestConfig(
                            RequestConfig.Builder requestConfigBuilder) {
                        // 连接超时（默认为1秒）
                        return requestConfigBuilder.setConnectTimeout(5000 * 1000)
                                .setSocketTimeout(6000 * 1000);
                    }
                });
        // 权限验证
        restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            UsernamePasswordCredentials credentials = new UsernamePasswordCredentials("xxx", "xxx");
            credentialsProvider.setCredentials(AuthScope.ANY, credentials);
            return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        });
        RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restClientBuilder);
        return restHighLevelClient;
    }

    private void syncData(String index, List<String> sourceList) throws IOException {
        // newClient是连接到待迁移的客户端
        RestHighLevelClient newClient = getNewClient();
        BulkRequest bulkRequest = new BulkRequest();
        // 超时时间
        bulkRequest.timeout(TimeValue.timeValueHours(24));
        for (String source : sourceList) {
            IndexRequest indexRequest = new IndexRequest(index);
            indexRequest.source(source, XContentType.JSON);
            bulkRequest.add(indexRequest);
        }
        newClient.bulk(bulkRequest, RequestOptions.DEFAULT);
    }