实现相同功能的两种代码格式，你们更喜欢哪种？

最新推荐文章于 2024-07-30 20:50:32 发布

啥也不会的大菜鸟

最新推荐文章于 2024-07-30 20:50:32 发布

阅读量177

点赞数

文章标签： java 开发语言

本文链接：https://blog.csdn.net/zhangchen66/article/details/131385207

版权

代码一：

@Override
    public void fetchData(String time) {
        try {
            log.info("开始拉取");
            String[] indexNames = new String[]{"a", "b", "c",
                    "d"};
            String[] includeFields = new String[]{"difld1", "difld2", "difld3", "difld4"};
            String storage_time = time;
            String path1 = "/home/data1/";
            String path2 = "/home/data2/";
            for (String indexName : indexNames) {
                log.info("开始拉取===>" + indexName);
                // 构建查询条件
                BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
                RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("storage_time").gte(storage_time);
                boolQuery.filter(rangeQuery);
                SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
                sourceBuilder.query(boolQuery);
                sourceBuilder.fetchSource(includeFields, null);
                sourceBuilder.sort("storage_time", SortOrder.DESC);
                sourceBuilder.size(10000);
                sourceBuilder.timeout(new TimeValue(60000));
                // 构建搜索请求
                SearchRequest searchRequest = new SearchRequest(indexName);
                searchRequest.source(sourceBuilder);
                searchRequest.scroll(new TimeValue(60000)); // 设置scroll参数
                // 执行搜索请求
                SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
                String scrollId = searchResponse.getScrollId(); // 获取第一批结果的scrollId
                SearchHit[] hits = searchResponse.getHits().getHits();
                // 处理搜索结果
                List<Map<String, Object>> dataList = new ArrayList<>();
                for (SearchHit hit : hits) {
                    dataList.add(hit.getSourceAsMap());
                }
                if (hits.length == 0) {
                    FileReadWriteUtil.fileWrite( path1+ indexName.substring(indexName.lastIndexOf("_") + 1) + ".txt", "");
                }
                // 获取下一批结果，直到没有更多结果为止
                while (hits != null && hits.length > 0) {
                    SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                    scrollRequest.scroll(new TimeValue(60000));
                    searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
                    scrollId = searchResponse.getScrollId();
                    hits = searchResponse.getHits().getHits();
                    for (SearchHit hit : hits) {
                        dataList.add(hit.getSourceAsMap());
                    }
                }
                ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
                clearScrollRequest.addScrollId(scrollId);
                client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); // 清除查询上下文
                // 将数据保存为json文件
                for (Map<String, Object> data : dataList) {
                    for (String a : includeFields) {
                        if (!data.containsKey(a)) {
                            data.put(a, null);
                        }
                    }
                    Map<String, Object> linkedHashMap = new LinkedHashMap<>(data);
                    String jsonString = new JSONObject(linkedHashMap).toString();
                    if (indexName.contains("others")) {
                        FileReadWriteUtil.fileWrite(path1+"otherData.txt", jsonString);
                    } else if (indexName.contains("attack")) {
                        FileReadWriteUtil.fileWrite(path1+"web.txt", jsonString);
                    } else if (indexName.contains("a")) {
                        FileReadWriteUtil.fileWrite(path2+"a.txt", jsonString);
                    } else {
                        FileReadWriteUtil.fileWrite(path1+ indexName.substring(indexName.lastIndexOf("_") + 1) + ".txt", jsonString);
                    }
                }
                log.info(indexName + "拉取完成");
            }
            log.info("全部拉取完成");
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

代码二：

/**
     * 需要拉取数据的es库
     */
    private final String[] indexNames = new String[]{"a", "b", "c",
                    "d"};

    /**
     * 各个库需要拉取的字段
     */
    private final String[] includeFields = new String[]{"difld1", "difld2", "difld3", "difld4"};

    private final String path1 = "/home/data1/";

    private final String path2 = "/home/data2/";

    private final long scrollTimeout = 60000;

    @Override
    public void fetchData(String time) {
        try {
            log.info("开始拉取");
            String startTime = time;
            for (String indexName : indexNames) {
                log.info("开始拉取===>" + indexName);
                SearchResponse searchResponse = searchWithScroll(indexName, startTime);
                List<Map<String, Object>> dataList = processSearchHits(searchResponse.getHits());
                if (searchResponse.getHits().getHits().length == 0) {
                    FileReadWriteUtil.fileWrite(path1 + indexName.substring(indexName.lastIndexOf("_") + 1) + ".txt", "");
                }
                while (searchResponse.getHits().getHits().length > 0) {
                    // 循环获取搜索结果，直到获取所有数据
                    searchResponse = scrollSearch(searchResponse.getScrollId());
                    dataList.addAll(processSearchHits(searchResponse.getHits()));
                }
                // 清除滚动上下文
                clearScrollContext(searchResponse.getScrollId());
                // 将数据保存为json文件
                saveDataAsJson(dataList, indexName);
                log.info(indexName + "拉取完成");
            }
            log.info("全部拉取完成");
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    private SearchResponse searchWithScroll(String indexName, String startTime) throws IOException {
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("storage_time").gte(startTime);
        boolQuery.filter(rangeQuery);

        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(boolQuery);
        sourceBuilder.fetchSource(includeFields, null);
        sourceBuilder.sort("storage_time", SortOrder.DESC);
        sourceBuilder.size(10000);
        sourceBuilder.timeout(new TimeValue(scrollTimeout));

        SearchRequest searchRequest = new SearchRequest(indexName);
        searchRequest.source(sourceBuilder);
        searchRequest.scroll(new TimeValue(scrollTimeout));

        return client.search(searchRequest, RequestOptions.DEFAULT);
    }

    private SearchResponse scrollSearch(String scrollId) throws IOException {
        SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
        scrollRequest.scroll(new TimeValue(scrollTimeout));
        return client.scroll(scrollRequest, RequestOptions.DEFAULT);
    }

    private void clearScrollContext(String scrollId) throws IOException {
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollId);
        client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
    }

    private List<Map<String, Object>> processSearchHits(SearchHits hits) {
        List<Map<String, Object>> dataList = new ArrayList<>();
        for (SearchHit hit : hits.getHits()) {
            Map<String, Object> data = hit.getSourceAsMap();
            for (String field : includeFields) {
                data.putIfAbsent(field, null);
            }
            dataList.add(new LinkedHashMap<>(data));
        }
        return dataList;
    }

    private void saveDataAsJson(List<Map<String, Object>> dataList, String indexName) throws IOException {
        for (Map<String, Object> data : dataList) {
            String jsonString = new JSONObject(data).toString();
            if (indexName.contains("others")) {
                FileReadWriteUtil.fileWrite(path1 +"otherData.txt", jsonString);
            } else if (indexName.contains("attack")) {
                FileReadWriteUtil.fileWrite(path1 +"web.txt", jsonString);
            } else if (indexName.contains("a")) {
                FileReadWriteUtil.fileWrite(path2 +"a.txt", jsonString);
            } else {
                FileReadWriteUtil.fileWrite(path1 + indexName.substring(indexName.lastIndexOf("_") + 1) + ".txt", jsonString);
            }
        }
    }