Java 超大文件（166G）读取

最近有个需求，将超级大的166GB 的csv文件读取解析并入库

废话少说，直接上干货：

走过路过~请点赞

/**
 * @author ff
 * @date 2022/1/6 19:16
 */
@Slf4j
public class FileReader {

    /**
     * 读文件
     *
     * @param filePath     文件路径
     * @param lineConsumer 行处理器
     * @throws IOException
     */
    public static void read(String filePath, Consumer<String> lineConsumer) throws IOException {
        final File csvFile = new File(filePath);
        if (!csvFile.exists()) {
            throw new FileNotFoundException("文件[" + filePath + "]不存在");
        }
        Long num = 0L;
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        try (
                FileInputStream fis = new FileInputStream(csvFile);
                Scanner sc = new Scanner(fis, "UTF-8")
        ) {
            while (sc.hasNext()) {
                String line = sc.nextLine();
                log.info("解析第[{}]行数据[{}]", ++num, line);
                if (Objects.nonNull(lineConsumer)) {
                    lineConsumer.accept(line);
                }
            }
            stopWatch.stop();
            log.info("CSV文件[{}]解析结束, 共用时：[{}]ms", filePath, stopWatch.getTotalTimeMillis());
        } catch (IOException ex) {
            log.error("解析CSV文件[{}]第[{}]行数据发生异常", filePath, num, ex);
        }
    }

    public static void main(String[] args) throws IOException {
        String file = "D:\\迅雷下载\\20220105SYQ.csv\\20220105SYQ.csv";
        FileReader.read(file, (line) -> {
			// 将line进行解析并入库
        });
    }
}