大文件IO流方式多线程分块、合并

网上看了一圈都是使用的零拷贝的方式进行分块，没有找到使用io流方式进行分块的，又因为公司某存储不支持零拷贝的使用，在此记录一下。

public class ChannelMain {

    /**
     * 单个分片大小
     */
    public static int PART_SIZE = 1024 * 1024 * 100;

    public static void main(String[] args) throws IOException {
        String srcPath = "H:\\part\\74一见倾心系列第七十四期（汉服）.zip";
        String outPath = "H:\\part";

//        splitBigFile(srcPath, outPath);
        String[] list = new File(outPath).list();
        List<String> fileList = new CopyOnWriteArrayList<>();
        assert list != null;
        for (String data : list) {
            fileList.add(outPath + File.separator + data);
        }
        fileList = fileList.stream().sorted((i1, i2) -> {
            Integer s1 = Integer.parseInt(i1.split("_")[1]);
            Integer s2 = Integer.parseInt(i2.split("_")[1]);
            return s1.compareTo(s2);
        }).collect(Collectors.toList());
        mergeBigFile(fileList, srcPath);
    }

    /**
     * 大文件合并
     */
    private static void mergeBigFile(List<String> srcPathList, String outPath) throws IOException {
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        RandomAccessFile rFile = new RandomAccessFile(outPath, "rw");
        try {
            long startIndex = 0L;
            for (String srcPartPath : srcPathList) {
                rFile.seek(startIndex);
                FileInputStream fileInputStream = null;
                try {
                    fileInputStream = new FileInputStream(srcPartPath);
                    int available = fileInputStream.available();
                    byte[] bytes = new byte[available];
                    fileInputStream.read(bytes);
                    //写入
                    rFile.write(bytes);
                    startIndex += available;

                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    if (fileInputStream != null) {
                        fileInputStream.close();
                    }
                }
            }
        } finally {
            rFile.close();
        }
        stopWatch.stop();
        System.out.println("耗时=" + stopWatch.getTotalTimeMillis() + "ms");
    }


    /**
     * 大文件分割
     *
     * @param srcPath
     * @param outPath
     * @throws IOException
     */
    private static void splitBigFile(String srcPath, String outPath) throws IOException {
        RandomAccessFile rFile = new RandomAccessFile(srcPath, "r");
        long fileAllSize = rFile.length();
        int count = (int) (fileAllSize / PART_SIZE);
        String partRootPath = outPath;
        if (fileAllSize % PART_SIZE != 0) {
            count++;
        }
        System.out.println("[分块为] " + fileAllSize + ", allPartNums=" + count);
        int k = 1;
        long startIndex = 0;
        ExecutorService executorService = Executors.newFixedThreadPool(5);
        CopyOnWriteArrayList<CompletableFuture<Void>> arrayList = new CopyOnWriteArrayList<>();
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        while (k <= count) {
            String partItemPath = partRootPath + File.separator + "_" + k;
            File file = new File(partItemPath);
            if (!file.exists()) file.createNewFile();
            long finalStartIndex = startIndex;
            CompletableFuture<Void> runAsync = CompletableFuture.runAsync(() -> {
                FileOutputStream outputStream = null;
                try {
                    byte[] bytes = new byte[(int) PART_SIZE];
                    outputStream = new FileOutputStream(file);
                    //设置读取的位置
                    // 注意 rFile.seek 不是线程安全的 所以需要锁定该对象
                    synchronized (rFile) {
                        rFile.seek(finalStartIndex);
                        //读取分片
                        int read = rFile.read(bytes);
                        if (read != PART_SIZE && read > 0) {
                            //调整最后的大小
                            bytes = getActualBytes(bytes, read);
                        }
                    }
                    outputStream.write(bytes);

                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        if (outputStream != null) {
                            outputStream.flush();
                            outputStream.close();
                        }
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }, executorService);
            arrayList.add(runAsync);
            startIndex += PART_SIZE;
            k++;
        }
        arrayList.forEach(CompletableFuture::join);
        rFile.close();
        stopWatch.stop();
        System.out.println("耗时=" + stopWatch.getTotalTimeMillis() + "ms");
    }

    /**
     * 截取真实长度的数组.
     *
     * @param bytes
     * @param read
     * @return
     */
    private static byte[] getActualBytes(byte[] bytes, int read) {
        byte[] b1 = new byte[read];
        System.arraycopy(bytes, 0, b1, 0, read);
        return b1;
    }

}

 

 测试完成，合并后压缩包可正确打开。