大文件json转储mongoDB 内存不溢出

1.需要在页面下载zip文件进行解压

 /**
     * 远程文件下载地址
     *
     * @param filePath 网络文件请求地址
     */
    public static void downAndReadFile(String filePath,String dir,String data) {
        long startTime = System.currentTimeMillis();
        // 声明文件对象
        File saverPath = new File(dir);
        // 判断文件是否存在
        // 判断文件是否存在
        if (!saverPath.exists()) {
            // 文件不存在就创建一个一级目录【远程请求下载】
            saverPath.mkdir();
        }
        // 根据/切割接受到的请求网络URL
        String[] urlName = filePath.split("/");
        // 获取到切割的字符串数组长度-1
        int len = urlName.length - 1;
        // 获取到请求下载文件的名称
        String uname = urlName[len];

        BufferedOutputStream bufferedOutputStream = null;
        InputStream inputStream = null;
        BufferedInputStream fs = null;
        BufferedReader br = null;
        // 跳过try捕获错误
        try {
            // 创建保存文件对象
            File file = new File(saverPath + "\\" + uname);//创建新文件
            if (file != null && !file.exists()) {
                file.createNewFile();
            }
            // 通过高效字节输出流输出创建的文件对象
            bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file));
            // 创建URL对象[请求路径]
            URL url = new URL(filePath);
            // 返回一个URLConnection实例，表示与URL引用的远程对象的URL
            HttpURLConnection uc = (HttpURLConnection) url.openConnection();
            uc.setDoInput(true); // 设置的值 doInput领域本 URLConnection指定值。
            uc.connect(); // 打开与此URL引用的资源的通信链接，如果此类连接尚未建立。
            // 获取服务端的字节输入流
            inputStream = uc.getInputStream();
            System.out.println("file gz is：" + uc.getContentLength()); // 打印文件的长度
            // 声明字节数组存放读取的文件
            byte[] b = new byte[1024 * 1024 * 5];
            int byteRead = -1; // 定义读取次数
            // 循环读取
            while ((byteRead = inputStream.read(b)) != -1) {
                bufferedOutputStream.write(b, 0, byteRead); // 将读取的文件跳过高效的字节流输出
            }

            long endTime = System.currentTimeMillis();
            System.out.println("下载耗时：" + (endTime - startTime) / 1000 * 1.0 + "s");
            System.out.println("文件下载成功！");

            // ---------- 解压文件 ----------
            StringBuffer strb = new StringBuffer();
            // 创建高效的字节输入管道
            fs = new BufferedInputStream(new FileInputStream(saverPath + "\\" + uname));
            br = new BufferedReader(new InputStreamReader(fs, "UTF-8")); // 指定读取的编码格式); // 高效缓存字节读取
            String date = ""; // 记录读取一行的数据
            // 循环读取
            while ((date = br.readLine()) != null) {
                strb.append(data + " "); // 将读取的数据赋值给可变的字符串
            }
            System.out.println("解压文件中...");
            //解压
            unGzFiles(dir + "\\" + uname, dir + "\\");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            // 关闭流和刷新流
            try {
                inputStream.close();
            } catch (IOException ioException) {
                ioException.printStackTrace();
            }
            try {
                bufferedOutputStream.close();
            } catch (IOException ioException) {
                ioException.printStackTrace();
            }
            try {
                br.close();
            } catch (IOException ioException) {
                ioException.printStackTrace();
            }
            try {
                fs.close();
            } catch (IOException ioException) {
                ioException.printStackTrace();
            }
        }
    }

    /**
     * 解压的文件
     *
     * @param gzPath 压缩文件
     * @param descDir 解压存放的位置
     * @throws Exception
     */
    public static void unGzFiles(String gzPath, String descDir)  {
        System.out.println("解压文件的名称：" + gzPath + "解压的文件存放路径：" + descDir);
        upGzFiles(gzPath); // 调用方法
    }

    /**
     * 解压文件到指定的位置
     *
     * @param gzPath 解压文件
     */
    @SuppressWarnings("rawtypes") // 抑制警告【原始类型】
    public static void upGzFiles(String gzPath) {
        String ouputfile = "";
        FileInputStream fileInputStream = null;
        GZIPInputStream gzipInputStream = null;
        FileOutputStream fileOutputStream = null;
        try {
            //建立gzip压缩文件输入流
            fileInputStream = new FileInputStream(gzPath);
            //建立gzip解压工作流
            gzipInputStream = new GZIPInputStream(fileInputStream);
            //建立解压文件输出流
            ouputfile = gzPath.substring(0, gzPath.lastIndexOf('.'));
            fileOutputStream = new FileOutputStream(ouputfile);

            int num;
            byte[] buf = new byte[1024];

            while ((num = gzipInputStream.read(buf, 0, buf.length)) != -1) {
                fileOutputStream.write(buf, 0, num);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (gzipInputStream != null) {
                try {
                    gzipInputStream.close();
                } catch (IOException ioException) {
                    ioException.printStackTrace();
                }
            }
            if (fileOutputStream != null) {
                try {
                    fileOutputStream.close();
                } catch (IOException ioException) {
                    ioException.printStackTrace();
                }
            }
            if (fileInputStream != null) {
                try {
                    fileInputStream.close();
                } catch (IOException ioException) {
                    ioException.printStackTrace();
                }
            }
        }
    }

2.通过工具类实现json转换对象 1000条存储一次 内存不会溢出 

package com.originspace.env.utils;
 
 
import com.alibaba.excel.util.ListUtils;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MappingJsonFactory;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.function.Consumer;


public class ParseJsonUtil {
    private static int BATCH_COUNT = 1000;

    public static <T> List<T> getOrderObjectsByClass(String jsonPath, Class<T> type, Consumer<List<T>> consumer) throws IOException {
        List<T> returnList = new LinkedList<T>();
        ObjectMapper objectMapper = new ObjectMapper();
        JsonFactory f = new MappingJsonFactory();
        JsonParser jp = f.createParser(new File(jsonPath));
        JsonToken current;
        current = jp.nextToken();
        if (current != JsonToken.START_ARRAY) {
            System.out.println("Error: root should be object: quiting.");
            return null;
        }
        while (current != JsonToken.END_ARRAY) {
            current = jp.nextToken();
            while (current == JsonToken.START_OBJECT) {
                current = jp.nextToken();
                JsonNode node = jp.readValueAsTree();
                String s = objectMapper.writeValueAsString(node);
                T mpcorbExtend = JSONObject.parseObject(s, type);
                returnList.add(mpcorbExtend);
                if (returnList.size() >= BATCH_COUNT) {
                    consumer.accept(returnList);
                    returnList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
                }
            }
        }
        if (returnList != null) {
            consumer.accept(returnList);
        }
        return returnList;
    }

}

此处借鉴easyexcel中的写法  1000条返回一次

easyexcel解析CSV文件

 public void readAndSaveNasaDataByOther(String fileName){
        EasyExcel.read(fileName, NasaData.class, new PageReadListener<NasaData>(dataList -> {
        datalist->{//一系列操作}
        })).sheet().doRead();
    }