大文件读取java代码

/**
 * 全局配置文件上传大小
 * 
 */
@Bean
public MultipartConfigElement multipartConfigElement() {
    MultipartConfigFactory factory = new MultipartConfigFactory();
    //文件最大
    factory.setMaxFileSize("100MB"); //KB,MB
    /// 设置总上传数据总大小
    factory.setMaxRequestSize("100MB");
    return factory.createMultipartConfig();
}

 

//大文件读取处理方式一：

package com.zhongan.xd.util.test;

import com.google.common.collect.Lists;
import com.zhongan.xd.bss.common.CommonConstants;
import com.zhongan.xd.bss.dal.model.abs.AbsAssetRelationDO;
import com.zhongan.xd.bss.util.MappedByteBufferReadLine;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;

import java.io.File;
import java.io.FileInputStream;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *
 */
@Slf4j
public class MappedByteBufferReadLineTest {

    private FileInputStream  fis;
    private FileChannel      fc;
    private MappedByteBuffer mbb;

    private int              currentReadPos;
    private int              limit;

    public MappedByteBufferReadLineTest(File file) throws Exception {
        if (!file.exists() || !file.isFile()) {
            throw new Exception("指定文件不存在或者不是一个文件");
        }
        fis = new FileInputStream(file);
        fc = fis.getChannel();
        mbb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
        limit = mbb.limit();
    }

    public MappedByteBufferReadLineTest(String filePath) throws Exception {
        this(new File(filePath));
    }

    /**
     * 指定每行的容量，最大字节数 如果存在行超过指定最大字节，则会
     *
     * @param capacity
     * @return
     * @throws Exception
     */
    public String readLine(int capacity) throws Exception {
        try {
            if (currentReadPos >= limit) {
                return null;
            }
            ByteBuffer bb = ByteBuffer.allocate(capacity == 0 ? 1024 : capacity);
            while (currentReadPos < limit) {
                byte b = mbb.get();
                currentReadPos++;
                if (System.getProperty("line.separator").equals("\r\n") && b == 13) {
                    mbb.get();
                    currentReadPos++;
                    break;
                } else if (b == 10 || b == 13) {
                    break;
                } else {
                    bb.put(b);
                }
            }
            return rightTrim(new String(bb.array(), "UTF-8"));
        } catch (Exception e) {
            clean();
            throw e;
        } finally {
            fc.close();
            fis.close();
        }
    }

    /**
     * 默认1024字节
     *
     * @return
     * @throws Exception
     */
    public String readLine() throws Exception {
        return readLine(0);
    }
    /**
     * 清理ByteBuffer
     *
     * @param
     * @throws Exception
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    public void clean() throws Exception {
        AccessController.doPrivileged(new PrivilegedAction() {
            @Override
            public Object run() {
                try {
                    Method getCleanerMethod = mbb.getClass().getMethod("cleaner", new Class[0]);
                    getCleanerMethod.setAccessible(true);
                    sun.misc.Cleaner cleaner = (sun.misc.Cleaner) getCleanerMethod.invoke(mbb, new Object[0]);
                    cleaner.clean();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return null;
            }
        });
    }

    private String rightTrim(String s) {
        char[] cs = s.toCharArray();
        int pos = 0;
        for (int i = cs.length - 1; i >= 0; i--) {
            String tostr = String.valueOf(cs[i]);
            if (tostr.trim().length() != 0) {
                pos = i;
                break;
            }
        }
        return s.substring(0, pos + 1);
    }

    public static void main(String[] args) throws Exception {
        Long start=System.currentTimeMillis();
        List<AbsAssetRelationDO> repayNoList = Lists.newArrayList();
        MappedByteBufferReadLineTest mbbrlutil = null;
        int temp=0;
        int countdown=3000;
        try {
            mbbrlutil = new MappedByteBufferReadLineTest("D:\\迭代\\20200922\\cesDat.csv");
            temp = 0;
            countdown = 3000;
            String lineFirst = mbbrlutil.readLine();
            log.info("lineFirst=={}", lineFirst);
            String line = null;
            while ((line = mbbrlutil.readLine()) != null) {
                line = line.replaceAll("\"", "");
                Pattern p = Pattern.compile("\\s*|\t|\r|\n");
                Matcher m = p.matcher(line);
                String dest = m.replaceAll("");
                AbsAssetRelationDO absAssetRelationDO=new AbsAssetRelationDO();
                absAssetRelationDO.setRelationNo(dest);
                if (temp > countdown) {
                    countdown = temp + 3000;
                    //执行插入
                    log.info("countdown=={},temp=={}  spendTime=={}  dataSize==",countdown, temp,System.currentTimeMillis() - start,repayNoList.size());
                    repayNoList = null;
                    repayNoList = Lists.newArrayList();
                }
                temp++;
            }
            log.info("temp=={} ",temp);
            if(CollectionUtils.isNotEmpty(repayNoList)){
                log.info("spendTime=={}  dataSize==", System.currentTimeMillis() - start,repayNoList.size());

            }
        System.out.println(System.currentTimeMillis()-start);

    }catch (Exception e){
        }
    }
}

 

 

//大文件读取处理方式二

 

public static void main(String[] args) throws IOException {
    Long start = System.currentTimeMillis();
    File file = new File("D:\\迭代\\20200922\\cesDat.csv");
    FileInputStream inputStream = null;
    Scanner sc = null;
    try {
        inputStream = new FileInputStream(file);
        sc = new Scanner(inputStream, "UTF-8");
        while (sc.hasNextLine()) {
           String line= sc.nextLine();
            log.info("line Time=={}", line);
        }

    } catch (Exception e) {
        log.info("失败信息Exception=={}", e);
    } finally {
        if (inputStream != null) {
            inputStream.close();
        }
        if (sc != null) {
            sc.close();
        }
    }
    log.info("assembleDate Time=={}", System.currentTimeMillis() - start);
}

//批处理文件每隔一定的数量插入数据

 

public void assembleDate(File file, AbsPackUploadRequest request) {
    List<AbsAssetRelationDO> repayNoList = Lists.newArrayList();
    Long start = System.currentTimeMillis();
    MappedByteBufferReadLine mbbrlutil = null;
    try {
        mbbrlutil = new MappedByteBufferReadLine(file);
        int temp = 0;
        int countdown = 3000;
        String lineFirst = mbbrlutil.readLine();
        log.info("lineFirst=={}", lineFirst);
        fileSave(request);
        String line = null;
        while ((line = mbbrlutil.readLine()) != null) {
            if (StringUtils.isNotBlank(line)) {
                AbsAssetRelationDO absAssetRelationDO = new AbsAssetRelationDO();
                absAssetRelationDO.setId(seqAbsAssetRelation.nextValue());
                absAssetRelationDO.setRelationNo(line);
                repayNoList.add(absAssetRelationDO);
            }
            if (temp > countdown) {
                countdown = temp + 3000;
                //执行插入
                saveAssetRelation(request, repayNoList);
                 repayNoList = null;
                repayNoList = Lists.newArrayList();
            }
            temp++;
        }
        if (CollectionUtils.isNotEmpty(repayNoList)) {
            saveAssetRelation(request, repayNoList);
        }
        log.info("temp=={}", temp);
        file.delete();
        log.info("assembleDate Time=={}", System.currentTimeMillis() - start);
    } catch (Exception e) {
        log.info("失败信息Exception=={}", e);
    } finally {
        try {
            if (null != mbbrlutil) {
                mbbrlutil.clean();
            }
        } catch (Exception e) {
            log.error("assembleDateCloseFailed e=={}", e);
        }
    }
}