restHighLevelClient客户端连接elasticsearch实现数据迁移

可以问吗

已于 2024-07-26 11:17:25 修改

阅读量260

点赞数 2

文章标签： elasticsearch 大数据搜索引擎

于 2024-07-25 22:11:37 首次发布

本文链接：https://blog.csdn.net/binbushi/article/details/140701202

版权

restHighLevelClient客户端连接elasticsearch实现数据迁移

依赖

	<dependencies>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>7.15.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.2</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.83</version>
        </dependency>
        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.7.22</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.10.0</version>
        </dependency>
    </dependencies>

resouces目录下的log4j2.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration>
    <Appenders>
        <Console name="STDOUT" target="SYSTEM_OUT">
            <PatternLayout pattern="%d %-5p [%t] %C{2} (%F:%L) - %m%n"/>
        </Console>
    </Appenders>
    <Loggers>
        <Logger name="com.opensymphony.xwork2" level="WAN"/>
        <Logger name="org.apache.struts2" level="WAN"/>
        <Root level="warn">
            <AppenderRef ref="STDOUT"/>
        </Root>
    </Loggers>

</Configuration>

类

import com.alibaba.fastjson.JSON;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;

import java.io.IOException;
import java.util.*;

/**
 *
 */
public class ESTestClient {

    //只选自己需要的索引，多余的官方索引复制到自己的es会导致自己的es可能故障
    //以.开头的通常是官方索引
    public static final String[] indexList = new String[]
            {"community", "fin_bill", "fin_bill_detail", "fin_bill_flow", "house", "house_book", "house_check_in",
                    "owner_contract", "platform_building_house", "platform_house", "platform_house_fk", "renter_contract"};


    //！！！！注意客户端的地址不能错误，不要将本地数据插入到来源es中

    /**
     * 想要插入数据的客户端
     *
     * @return
     */
    public RestHighLevelClient getTargetClient() {
        //创建ES客户端
        RestHighLevelClient esClient = new RestHighLevelClient(
                RestClient.builder(new HttpHost("192.168.200.129", 9200, "http"))
        );
        return esClient;
    }

    /**
     * 获取数据的服务器
     *
     * @return
     */
    public RestHighLevelClient getResourceClient() {
        //创建ES客户端
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("admin", "xxxxx"));
        RestClientBuilder httpBuilder = RestClient.builder(new HttpHost("192.168.1.2", 9200, "http"));
        httpBuilder.setHttpClientConfigCallback(httpconfig -> {
            httpconfig.setDefaultCredentialsProvider(credentialsProvider);
            return httpconfig;
        });
        //创建ES客户端
        RestHighLevelClient esClient = new RestHighLevelClient(httpBuilder);
        return esClient;
    }

    /**
     * 获取全部索引
     *
     * @throws IOException
     */
    @Test
    public void getAllIndexList() throws IOException {
//        RestHighLevelClient esClient = getTargetClient();
        RestHighLevelClient esClient = getResourceClient();
        GetIndexRequest request = new GetIndexRequest("*");
        GetIndexResponse response = esClient.indices().get(request, RequestOptions.DEFAULT);
        //获取到全部索引名称
        String[] indices = response.getIndices();
        System.out.println("索引数量:" + indices.length);
        //以.开头的是官方索引，不要进行操作
        System.out.println("------------------正常索引----------------------");
        for (String index : indices) {
            if (!index.startsWith(".")) {
                System.out.println(index);
            }
        }
        System.out.println("------------------官方索引----------------------");
        for (String index : indices) {
            if (index.startsWith(".")) {
                System.out.println(index);
            }
        }
    }

    //
    //全部索引里面有很多是不需要的官方索引和多余索引
    @Test
    public void getAllMapping() throws IOException {
//        RestHighLevelClient esClient = getTargetClient();
        RestHighLevelClient esClient = getResourceClient();
        //获取来源es的索引和映射
        GetIndexRequest request = new GetIndexRequest(indexList);
        GetIndexResponse response = esClient.indices().get(request, RequestOptions.DEFAULT);
        Map<String, MappingMetadata> mappings = response.getMappings();
        for (Map.Entry<String, MappingMetadata> entry : mappings.entrySet()) {
            //只设置properties
            Map<String, Object> sourceAsMap = entry.getValue().getSourceAsMap();
            Object properties = sourceAsMap.get("properties");
            Map<String, Object> propertiesMap = new HashMap<>();
            propertiesMap.put("properties", properties);
            String mapping = JSON.toJSONString(propertiesMap);
            System.out.println(entry.getKey());
            System.out.println(mapping);
        }
    }

    /**
     * 获取索引中的数据
     *
     * @throws IOException
     */
    @Test
    public void getDocumentList() throws IOException {
        RestHighLevelClient esClient = getTargetClient();
//        RestHighLevelClient resourceClient = getResourceClient();
        for (String index : indexList) {
            SearchRequest searchRequest = new SearchRequest();
            //设置参数 --- 表示查询哪个索引中的文档内容
            searchRequest.indices(index);
            //构建查询的请求体 --- 存入搜索请求对象中
            SearchSourceBuilder builder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
            builder.from(0);
            builder.size(10);
//            builder.size(2000);

//            builder.sort("age", SortOrder.ASC);
            //将构建好的查询请求体存入搜索请求对象中
            searchRequest.source(builder);
            SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);

            SearchHits hits = searchResponse.getHits();
            System.out.println(index);//索引名称
            System.out.println(hits.getTotalHits()); //结果集的条数
            System.out.println(searchResponse.getTook());  //总耗时
            for (SearchHit hit : hits) {
                System.out.println(hit.getSourceAsString());
            }
        }
    }

    /**
     * 复制索引和映射
     *
     * @throws IOException
     */
    //
    @Test
    public void copyIndexAndMapping() throws IOException {
        //目的es
        RestHighLevelClient resourceClient = getTargetClient();
        //来源es
        RestHighLevelClient targetClient = getResourceClient();
        //获取来源es的索引和映射
        GetIndexRequest request = new GetIndexRequest(indexList);
        GetIndexResponse response = targetClient.indices().get(request, RequestOptions.DEFAULT);
        Map<String, MappingMetadata> mappings = response.getMappings();
        for (Map.Entry<String, MappingMetadata> entry : mappings.entrySet()) {
            System.out.println(entry.getKey());
            //只设置properties
            Map<String, Object> sourceAsMap = entry.getValue().getSourceAsMap();
            Object properties = sourceAsMap.get("properties");
            Map<String, Object> propertiesMap = new HashMap<>();
            propertiesMap.put("properties", properties);
            String mapping = JSON.toJSONString(propertiesMap);
            CreateIndexRequest createIndexRequest = new CreateIndexRequest(entry.getKey()).mapping(mapping, XContentType.JSON);
            resourceClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
        }
    }

    /**
     * 每个文档只复制2000条  多了内存不够
     *
     * @throws IOException
     */
    @Test
    public void copyDocument() throws IOException {
        //本地es
        RestHighLevelClient targetClient = getTargetClient();
        //公司es
        RestHighLevelClient resourceClient = getResourceClient();

        for (String index : indexList) {
            SearchRequest searchRequest = new SearchRequest();
            //设置参数 --- 表示查询哪个索引中的文档内容
            searchRequest.indices(index);
            //构建查询的请求体 --- 存入搜索请求对象中
            SearchSourceBuilder builder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery());
            builder.from(0);
            builder.size(2000);

//            builder.sort("age", SortOrder.ASC);
            //将构建好的查询请求体存入搜索请求对象中
            searchRequest.source(builder);
            SearchResponse searchResponse = resourceClient.search(searchRequest, RequestOptions.DEFAULT);

            SearchHits hits = searchResponse.getHits();
            System.out.println(index);//索引名称
            System.out.println(hits.getTotalHits()); //结果集的条数
            System.out.println(searchResponse.getTook());  //总耗时
            BulkRequest bulkRequest = new BulkRequest();
            for (SearchHit hit : hits) {
                System.out.println(hit.getSourceAsString());
                IndexRequest indexRequest = new IndexRequest(index);
                indexRequest.source(hit.getSourceAsString(), XContentType.JSON);
                bulkRequest.add(indexRequest);
            }
            targetClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        }
    }
}