大批量数据分批式导出文件解决,避免OOM(多次查询多次导出形成一个文件)

大批量数据的导出,当数据量达到一定的量会导致内存被撑爆,出现 oom异常,基于问题实大批量数据分批的方式进行查询和导出

 

    

 

代码实现

package com.ly.service;

import com.ly.helper.BatchWriteFileUtils;
import com.ly.helper.BeanUtils;
import com.ly.vo.rsp.ClearRepaymentRspVo;
import lombok.extern.log4j.Log4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.io.File;
import java.io.IOException;
import java.util.List;

/**
 * @author fyc
 * @description: 加班后心血来潮分享一下, 代码为阉割版,如果不全,可电话联系到我 15611453880  付玉超
 * mail: fycstart@126.com
 * @date 2019/8/19下午 9:03
 */
@Service
@Log4j
public class ClearSettlementServiceImplTest {


    @Autowired
    @Qualifier(value = "TODAY_HAS_BEEN_CLEARED")
    private GetDataHandler todayHasBeenClearedGetDataHandler;


    //输出文件方法
    public File createExcelByType() throws IOException {
        File csvFile = null;
        String fileDir = "D://" + File.separator + "clear_settlement";
        //文件类型可自定义,修改输出方式即可
        String fileName = "test_ClearSettlement.csv";
        String filePath = fileDir + File.separator + fileName;

        List<Object> allFieldName = BeanUtils.getAllFieldName(ClearRepaymentRspVo.class);

        //核心方法 BeanUtils.getAllFieldName
        csvFile = BatchWriteFileUtils.createFile(allFieldName, filePath, todayHasBeenClearedGetDataHandler);

        return csvFile;
    }
}

 

 

package com.ly.helper;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;

/**
 * @author fyc
 * @description: 操作实体
 * @date 2019/10/21下午 5:38
 */
public class BeanUtils {

    private static final Logger LOGGER = LoggerFactory.getLogger(BeanUtils.class);


    public static List<Object> getAllFieldName(Class clazz) {
        List<Object> fields = new ArrayList<>();
        while (clazz != null) {
            for (Field declaredField : clazz.getDeclaredFields()) {
                String name = declaredField.getName();
                fields.add(name);
            }
            clazz = clazz.getSuperclass();
        }
        return fields;
    }


    public static List<Object> getAllFieldValue(Object model) {
        Class clazz = model.getClass();
        List<Object> fields = new ArrayList<>();
        while (clazz != null) {
            Object obj = null;
            for (Field declaredField : clazz.getDeclaredFields()) {
                String name = declaredField.getName();
                //属性名首字母大写
                Method m = null;
                try {
                    m = clazz.getMethod("get" + name.substring(0, 1).toUpperCase() + name.substring(1));
                } catch (NoSuchMethodException e) {
                    LOGGER.error("class: {} An exception occurred with the reflection fetch method {}", clazz.getName(), e);
                }
                // 调用getter方法获取属性值
                try {
                    obj = m.invoke(model);
                } catch (Exception e) {
                    LOGGER.error("class: {}  Getting value by reflection is an exception {}", clazz.getName(), e.toString());
                }
                fields.add(obj != null ? obj : "");
            }
            clazz = clazz.getSuperclass();
        }
        return fields;
    }
}

 

package com.ly.helper;

import com.ly.service.GetDataHandler;
import lombok.extern.slf4j.Slf4j;

import java.io.*;
import java.util.List;
import java.util.stream.Collectors;

@Slf4j
public class BatchWriteFileUtils {


    /**
     * 创建文件,写入文件头
     *
     * @param outPutPath
     * @return
     * @throws IOException
     */
    public static <T> File createFile(List<Object> head, String outPutPath, GetDataHandler<T> getDataHandler) throws IOException {
        File csvFile = new File(outPutPath);
        File parent = csvFile.getParentFile();
        if (parent != null && !parent.exists()) {
            parent.mkdirs();
        }

        try (FileOutputStream fileOutputStream = new FileOutputStream(csvFile);
             OutputStreamWriter outputStreamWriter = new OutputStreamWriter(fileOutputStream, "GB2312");
             BufferedWriter buffWriter = new BufferedWriter(outputStreamWriter, 1024);) {

            writeHeader(head, buffWriter);

            int number = 1;
            int size = 20000;
            while (true) {
                List<T> execute = getDataHandler.execute(number, size);
                if (execute == null || execute.size() < 1) {
                    break;
                }
                batchWriter(execute, buffWriter);
                number++;

            }
        } catch (Exception e) {
            throw new RuntimeException("FilePath:" + csvFile.getPath() + " Failed to get file stream ", e);
        }
        return csvFile;
    }


    /**
     * 写入文件头
     *
     * @param head
     * @return
     */
    private static void writeHeader(List<Object> head, BufferedWriter buffWriter) {
        // GB2312使正确读取分隔符","
        try {
            // 写入文件头部
            writeRow(head, buffWriter);
            buffWriter.flush();
        } catch (Exception e) {
            throw new RuntimeException("Write file error", e);
        }
    }


    /**
     * 写入文件体
     *
     * @param dataList
     * @throws IOException
     */
    private static <T> void batchWriter(List<T> dataList, BufferedWriter csvWriter) throws IOException {

        List<List<Object>> collect = dataList.parallelStream().map(item -> {
            List<Object> allFieldValue = BeanUtils.getAllFieldValue(item);
            return allFieldValue;
        }).collect(Collectors.toList());


        try {
            // 写入文件内容
            for (List<Object> row : collect) {
                writeRow(row, csvWriter);
            }
            csvWriter.flush();
        } catch (Exception e) {
            throw new RuntimeException("An exception occurred while writing the file tile", e);

        }
    }


    /**
     * 写一行数据方法
     *
     * @param row
     * @param csvWriter
     * @throws IOException
     */
    private static void writeRow(List<Object> row, BufferedWriter csvWriter) throws IOException {

        for (int i = 0; i < row.size(); i++) {
            StringBuffer sb = new StringBuffer();
            if (i < row.size() - 1) {
                sb.append("\"").append(row.get(i)).append("\",").toString().replace("null", "");
            } else {
                sb.append("\"").append(row.get(i)).append("\"").append("\r\n").toString().replace("null", "");
            }
            String aNull = sb.toString().replace("null", "");
            csvWriter.write(aNull);
        }
    }
}

 

 

package com.ly.service;

import java.io.IOException;
import java.util.List;

/**
 * @author fyc
 * @description: 对外暴露接口, 具体业务自行实现
 * @date 2019/10/22下午 7:08
 */
public interface GetDataHandler<T> {

    /**
     * @param number 页数
     * @param size   每页数量
     * @return
     */
    List<T> execute(int number, int size) throws IOException;
}

 

package com.ly.service;

import com.ly.helper.Result;

import java.util.List;
import java.util.Objects;

/**
 * @author fyc
 * @description: 对外暴露接口, 具体业务自行实现
 * @date 2019/10/22下午 7:08
 */
public abstract class AbstractHandlerData<T> implements GetDataHandler<T> {


    /**
     * @param result       需要统一处理集合
     * @param errorMessage 错误提示信息
     * @return
     */
    protected List<T> handlerData(Result<List<T>> result, String errorMessage) {
        if (result != null && Objects.equals(Result.Status.SUCCESS.code(), result.getState())) {
            List<T> data = result.getData();
            if (data != null && data.size() > 0) {
                return data;
            }
        } else {
            throw new RuntimeException(errorMessage);
        }
        return null;
    }


}

 

package com.ly.service.impl;

import com.ly.service.GetDataHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.List;

/**
 * @author fyc
 * @description: 查询今日还款数据
 * @date 2019/10/22下午 7:46
 */
@Component(value = "TODAY_HAS_BEEN_CLEARED")
public class TodayHasBeenClearedGetDataHandlerImpl implements GetDataHandler<TestVo> {

    @Autowired
    private TestDao testDao;


    @Override
    public List<TestVo> execute(int number, int size) {
        List<TestVo> testVos = testDao.queryData(number, size);
        return testVos;
    }

}