ES批量导入数据接口（es-dump文件，json文件，json数据）

话不多说直接上代码！

@Data
public class ImportRequestVO {
    private String index;
    /**
     * json文件路径
     */
    private String path;
    /**
     * dump文件路径
     */
    private String dumpPath ;
    /**
     * json数据
     */
    private List<Map<String, Object>> data;
}

	/**
     * 导入数据
     */

    @RequestMapping(value = "/importEsData", method = RequestMethod.POST)
    BsApiResponse importEsData(@Valid @RequestBody ImportRequestVO importRequestVO);

@Override
    public BsApiResponse importEsData(@Valid ImportRequestVO importRequestVO) {
        BsApiResponse bsApiResponse = new BsApiResponse();
        String path = importRequestVO.getPath();
        String dumpPath = importRequestVO.getDumpPath();
        List<Map<String, Object>> data = importRequestVO.getData();
        String indexs = importRequestVO.getIndex();
        try {
            if (ObjectUtils.isNotEmpty(data)) {
                executor.submit(() -> {
                    esToolService.importEsDataByJson(indexs, data);
                    return null;
                });
            }
            if (StringUtils.isNotEmpty(path)) {
                String[] indexArr = indexs.split(",");
                for (String index : indexArr) {
                    executor.submit(() -> {
                        esToolService.importEsData(index, path + index + ".json");
                        return null;
                    });
                }
            }
            if (StringUtils.isNotEmpty(dumpPath)) {
                String[] indexArr = indexs.split(",");
                for (String index : indexArr) {
                    executor.submit(() -> {
                        esToolService.importEsDataByDump(index, dumpPath + index + ".json");
                        return null;
                    });
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        bsApiResponse.setMessage("success");
        bsApiResponse.setCode("200");
        return bsApiResponse;
    }

dump文件大数据分批按行导入

public void importEsDataByDump(String index, String path) {
        // 导入接口
//        int idBase = 130000cnFaker0;
        BulkRequest peopleBulkRequest = new BulkRequest();
        try (BufferedReader br = new BufferedReader(new FileReader(path))) {
            String line;
            int count = 0;
            while ((line = br.readLine()) != null) {
                Map<String, Object> map = JsonMappers.Normal.fromJson(line, Map.class);
                Map<String, Object> sourceMap = (Map<String, Object>) map.get("_source");

                peopleBulkRequest.add(new IndexRequest(index).id(sourceMap.get("id").toString()).
                        source(JsonMappers.Normal.toJson(sourceMap), XContentType.JSON));

                // 每 10000 条文档执行一次 bulk 操作
                if (++count % 10000 == 0) {
                    BulkResponse response = restHighLevelClient.bulk(peopleBulkRequest, RequestOptions.DEFAULT);
                    if (response.hasFailures()) {
                        log.info("======= error:{}", response.buildFailureMessage());
                    }
                    System.out.println("count = " + count);
                    // 清空 bulk 请求
                    peopleBulkRequest.requests().clear();
                }
            }

            // 执行剩余的 docs （最后不足 10000 的一批）
            if (peopleBulkRequest.numberOfActions() > 0) {
                BulkResponse response = restHighLevelClient.bulk(peopleBulkRequest, RequestOptions.DEFAULT);
                if (response.hasFailures()) {
                    log.info("======= error:{}", response.buildFailureMessage());
                }
            }
            log.info("import index:{} success", index);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

json导入

 public void importEsDataByJson(String index, List<Map<String, Object>> data) {
        BulkRequest peopleBulkRequest = new BulkRequest();
        for (Map<String, Object> map : data) {
            if (map.containsKey("id")) {
                peopleBulkRequest.add(new IndexRequest(index).id(map.get("id").toString()).
                        source(JsonMappers.Normal.toJson(map), XContentType.JSON));
            } else {
                log.info("id is null !!");
            }
        }
        try {
            restHighLevelClient.bulk(peopleBulkRequest, RequestOptions.DEFAULT);
            log.info("import index:{} success", index);
        } catch (IOException e) {
            e.printStackTrace();
            log.info("import index:{} error", index);
        }
    }

json文件导入

public void importEsData(String index, String path) {
        // 导入接口
//        int idBase = 130000cnFaker0;
        BulkRequest peopleBulkRequest = new BulkRequest();
        String content = FileUtil.readFileToString(path);
        content = content.replace("\uFEFF", "");
        content = content.trim();
        List<Map<String, Object>> contentList = JsonMappers.Normal.fromJson(content, List.class);
        for (int i = 0; i < contentList.size(); i++) {
            Map<String, Object> map = contentList.get(i);
            if (map.containsKey("id")) {
                
                peopleBulkRequest.add(new IndexRequest(index).id(map.get("id").toString()).
                        source(JsonMappers.Normal.toJson(map), XContentType.JSON));
            } else {
                log.info("id is null !!");
            }
        }
        try {
            BulkResponse userResponse = restHighLevelClient.bulk(peopleBulkRequest, RequestOptions.DEFAULT);
            log.info("import index:{} success", index);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

测试

http://ip:port/importEsData
1. json文件导入 path为机器路径
需要path路径下有  ${index}.json 这个文件 ；path 最后要以 / 结尾
{
   "index": "user,hotel,aviation,train",
   "path": "/home/es_data/es/"
}
2. es dump 文件导入
dumpPath：dump文件路径
{
   "index": "people_case,case_event,alarm_detail,people_track",
   "dumpPath": "/home/es_data/es/"
}
3. json导入
只能单个索引导入
{
    "index": "vehicle_view",
    "data": [
        {
            "id": "1",
            "deviceName": "高速卡口",
            "snapTime": "2023-05-15 12:07:42",
            "plateNo": "皖C60F58",
            "snapUrl": "http://yq01-aip-gpu-test16.yq01:8099/ac/search/doc/picture?name=vehicle-0gO7tQ.jpg",
            "snapBgUrl": "http://yq01-aip-gpu-test16.yq01:8099/ac/search/doc/picture?name=vehicle-0gO7tQ.jpg",
            "deviceAddress": "G2京沪高速K1187江苏沪宁高速G2京沪高速K1187+650由西向东公路主线卡口",
            "deviceLontLat": {
                "lat": 31.2976024261,
                "lon": 121.1650897553
            }
        },
        {
            "id": "2",
            "deviceName": "高速卡口",
            "snapTime": "2023-05-15 19:50:03",
            "plateNo": "皖C60F58",
            "snapUrl": "http://yq01-aip-gpu-test16.yq01:8099/ac/search/doc/picture?name=vehicle-0gO7tQ.jpg",
            "snapBgUrl": "http://yq01-aip-gpu-test16.yq01:8099/ac/search/doc/picture?name=vehicle-0gO7tQ.jpg",
            "deviceAddress": "合肥市蜀山区合高架-北出口城区路口卡口",
            "deviceLontLat": {
                "lat": 31.5376024261,
                "lon": 117.0050897553
            }
        }
    ]
}
查看是否插入成功
Get http://ip:port/index/_search