springboot elasticSearch之间的数据迁移

elasticSearch 迁移最主要的是解决elasticSearch的依赖冲突，还有就是两个数据库之间一定要互通，至少要在一个es中连接到另一个es数据库，不然都只能在内网中访问，只能通过中间数据库来转换。
如果只是单纯的从es转数据库，这种就很好解决了
下面是我所使用的依赖 springboot 版本是2.2.0

<properties>
    <java.version>1.8</java.version>
    <elasticSearch.version>6.5.4</elasticSearch.version>
</properties>

<dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>

    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.58</version>
    </dependency>

    <dependency>
        <groupId>org.projectlombok</groupId>
        <artifactId>lombok</artifactId>
    </dependency>


    <dependency>
        <groupId>org.elasticsearch</groupId>
        <artifactId>elasticsearch</artifactId>
        <version>${elasticSearch.version}</version>
    </dependency>
    <dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>transport</artifactId>
        <version>${elasticSearch.version}</version>
    </dependency>
    <dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>elasticsearch-rest-high-level-client</artifactId>
        <version>${elasticSearch.version}</version>
    </dependency>
    <dependency>
        <groupId>org.elasticsearch.plugin</groupId>
        <artifactId>transport-netty4-client</artifactId>
        <version>${elasticSearch.version}</version>
    </dependency>
</dependencies>

下面是具体代码实现：文中需要的地方已经用注释说明，如果有问题请评论联系

import com.alibaba.fastjson.JSON;
import com.example.demo.entity.ESContant;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;

/**
 * @Author: chen
 * @Date: 2019/10/25 15:01
 * @Description:
 */
@Service
public class ESService {


    public void changeAllData() {

        try {

            //ESContant 为常量类 记载了es的index和type 需要自己创建

            //1、用户记录
            this.changeData(ESContant.LTC_LIMIT_USER, ESContant.LTC_LIMIT_USER);
            System.out.println(ESContant.LTC_LIMIT_USER + "-----------成功---------");

            //2、学分记录
            this.changeData(ESContant.LTC_COURSE_CREDIT, ESContant.LTC_COURSE_CREDIT);
            System.out.println(ESContant.LTC_COURSE_CREDIT + "-----------成功---------");



        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    public List<String> getAndInsert(TransportClient oldClient, String index, String type, TransportClient newClient) throws UnknownHostException {

        List<String> learnRecords = null;

        //我们是全量导数据，如果需要导部分数据 在setQuery 中设置查询语句
        SearchResponse scrollResp = oldClient.prepareSearch(index).setTypes(type)
                .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
                .setScroll(new TimeValue(60000))
//                .setQuery(qb)
                .setSize(5000).get();
        do {
            learnRecords = new ArrayList<>();
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                String sourceAsString = hit.getSourceAsString();
                learnRecords.add(sourceAsString);
            }
            // 插入到新的es
            this.insertAll(newClient, learnRecords, index, type);

            scrollResp = oldClient.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
        } while (scrollResp.getHits().getHits().length != 0);

        oldClient.close();
        newClient.close();

        return learnRecords;
    }

    /**
     * 转移数据
     *
     * @param index
     * @param type
     * @throws UnknownHostException
     */
    public void changeData(String index, String type) throws UnknownHostException {

        //1、简历连接
        TransportClient newEsClent = getNewEsClent();
        TransportClient oldEsClent = getOldEsClent();
        // 2、创建index和mapping
        getIndexInfoAndCreate(oldEsClent, index, type, newEsClent, index, type);

        //3、转移数据
        this.getAndInsert(oldEsClent, index, type, newEsClent);
    }

    /**
     * 插入数据到新的数据库
     * @param newEsClent
     * @param oldAll
     * @param index
     * @param type
     * @return
     * @throws UnknownHostException
     */
    public int insertAll(TransportClient newEsClent, List<String> oldAll, String index, String type) throws UnknownHostException {

        if (!CollectionUtils.isEmpty(oldAll)) {
            BulkRequestBuilder bulkRequest = newEsClent.prepareBulk();
            for (String source : oldAll) {
                HashMap map = JSON.parseObject(source, HashMap.class);
                if (map != null && map.containsKey("id") && !Objects.isNull(map.get("id"))) {
                    String id = map.get("id").toString();
                    if (id.equals("null") || StringUtils.isEmpty(id)) {
                        continue;
                    }
                    //添加请求
                    bulkRequest.add(newEsClent.prepareIndex(index, type, id)
                            .setSource(source, XContentType.JSON));
                }

            }
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();

            int count = bulkResponse.getItems().length;
            System.out.println(index + "-------插入成功------------------>" + count);
            return count;
        } else {
            System.out.println(index + "-------无数据------------------>");
            return 0;
        }

    }

    /**
     * 将原始数据中的mapping自动创建到新的es中，如果失败，就先在新的es数据库创建
     * @param client_from 来源client
     * @param index_from 来源index
     * @param type_from 来源 type
     * @param client_to 新的es数据库的 client
     * @param index_to 新的es数据库的 index
     * @param type_to 新的es数据库的 type
     * @return
     */
    private static boolean getIndexInfoAndCreate(Client client_from, String index_from, String type_from,
                                                 Client client_to, String index_to, String type_to) {

        ImmutableOpenMap<String, MappingMetaData> mappings = client_from.admin().cluster().prepareState().execute()
                .actionGet().getState().getMetaData().getIndices().get(index_from).getMappings();
        String mapping = mappings.get(type_from).source().toString();

        System.out.println(mapping);

        client_to.admin().indices().prepareCreate(index_to).execute().actionGet();
        PutMappingRequest mappingnew = Requests.putMappingRequest(index_to).type(type_to)
                .source(mapping.replaceFirst(type_from, type_to), XContentType.JSON);
        client_to.admin().indices().putMapping(mappingnew);

        return true;

    }

    /**
     * 需要转移的es
     *
     * @return
     * @throws UnknownHostException
     */
    public TransportClient getOldEsClent() throws UnknownHostException {

        // client.transport.sniff 设为true 会自动寻找网络内的集群，如果报错则为集群之间是内网不能通过外网找到，设为false就不寻找
        //cluster.name：集群名，一定要正确
        Settings settings = Settings.builder().put("cluster.name", "test126127128")
                .put("client.transport.sniff", true).build();

        TransportClient transportClient = new PreBuiltTransportClient(settings);

        //es数据库的地址，多少个集群就连多少
        TransportAddress transportAddress1 = new TransportAddress(InetAddress.getByName("ip1"), 9300);
        TransportAddress transportAddress2 = new TransportAddress(InetAddress.getByName("ip2"), 9300);
        TransportAddress transportAddress3 = new TransportAddress(InetAddress.getByName("ip3"), 9300);


        transportClient.addTransportAddress(transportAddress1);
        transportClient.addTransportAddress(transportAddress2);
        transportClient.addTransportAddress(transportAddress3);

        return transportClient;

    }

    /**
     * 转移到的es
     *
     * @return
     * @throws UnknownHostException
     */
    public TransportClient getNewEsClent() throws UnknownHostException {

        Settings settings = Settings.builder().put("cluster.name", "elasticsearch")
                .put("client.transport.sniff", true).build();

        TransportClient transportClient = new PreBuiltTransportClient(settings);

        TransportAddress transportAddress1 = new TransportAddress(InetAddress.getByName("ip1"), 9300);
        TransportAddress transportAddress2 = new TransportAddress(InetAddress.getByName("ip2"), 9300);
        TransportAddress transportAddress3 = new TransportAddress(InetAddress.getByName("ip3"), 9300);

        transportClient.addTransportAddress(transportAddress1);
        transportClient.addTransportAddress(transportAddress2);
        transportClient.addTransportAddress(transportAddress3);

        return transportClient;

    }
}

亲测可用，如有问题，请留言讨论。