Java文件切片 & 切片合并代码

研究了下大文件的断点上传，写了个文件切片和切片合并的代码，使用了RandomAccessFile和MappedByteBuffer。

测试了下性能，速度还可以。测试中使用了1.18G的待分割文件，每片最大1M，切片+合并测试了几次，平均耗时为4.8S左右，用的机械硬盘。

代码共享如下，需要的同学拿走。

文件分割器：

package com.lonzh.fileutils;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author Keven split a src-filt into sevral small pieces. The src-file's size
 * if required to be smaller than miPartSize * Integer.MAX_VALUE
 */
public class FileSplitter {

    private final String msSrcFile, msDstDir, msOutFileBaseName;
    private int miPartSize = 1024 * 1024;
    private int miMaxThrPoolSize = 10;
    private ThreadPoolExecutor moThrPool;

    private FileInputStream moInput;
    private FileChannel moInChannel;

    /**
     * Constructor
     *
     * @param psSrcFile
     * @param psDstDir
     * @param psOutFileBaseName the output files will be named by
     */
    public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName) {
        msSrcFile = psSrcFile;
        msDstDir = psDstDir;
        msOutFileBaseName = psOutFileBaseName;
    }

    /**
     * Constructor
     *
     * @param psSrcFile
     * @param psDstDir
     * @param psOutFileBaseName
     * @param piPartSize the split-out file's max-size
     */
    public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize) {
        msSrcFile = psSrcFile;
        msDstDir = psDstDir;
        msOutFileBaseName = psOutFileBaseName;
        miPartSize = piPartSize;
    }

    /**
     * Constructor
     *
     * @param psSrcFile
     * @param psDstDir
     * @param psOutFileBaseName
     * @param piPartSize
     * @param piMaxThrPool the max-size of thread pool
     */
    public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize, int piMaxThrPool) {
        msSrcFile = psSrcFile;
        msDstDir = psDstDir;
        msOutFileBaseName = psOutFileBaseName;
        miPartSize = piPartSize;
        miMaxThrPoolSize = piMaxThrPool;
    }

    /**
     * Judge if the splitter is working
     *
     * @return
     */
    public boolean isBusy() {
        return moThrPool != null && !moThrPool.isTerminated();
    }

    /**
     * Start splitting
     *
     * @return the array of the names of split-out files
     * @throws IOException
     */
    public String[] start() throws IOException {
        File loFile = new File(msSrcFile);
        if (!loFile.exists()) {
            throw new IOException("Src-File not found:" + msSrcFile);
        }

        long liFileLen = loFile.length();
        long liPartCnt = liFileLen / miPartSize + (liFileLen % miPartSize == 0 ? 0 : 1);
        if (liPartCnt > Integer.MAX_VALUE) {
            throw new IOException("Src-File too large");
        }

        moInput = new FileInputStream(msSrcFile);
        moInChannel = moInput.getChannel();
        moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
        String[] laOutFiles = new String[(int) liPartCnt];
        for (int i = 0; i < (int) liPartCnt; i++) {
            moThrPool.execute(new SplitThread(i, i == liPartCnt - 1 ? liFileLen % miPartSize : miPartSize));
            laOutFiles[i] = msDstDir + File.separator + msOutFileBaseName + "." + i + ".part";
        }
        moThrPool.shutdown();
        // Monitor
        new Thread() {
            @Override
            public void run() {
                while (true) {
                    if (moThrPool.isTerminated()) {
                        if (moInChannel != null) {
                            try {
                                moInChannel.close();
                            } catch (IOException ex) {
                                Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                            }
                        }
                        if (moInput != null) {
                            try {
                                moInput.close();
                            } catch (IOException ex) {
                                Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                            }
                        }
                        break;
                    }
                    try {
                        Thread.sleep(100);
                    } catch (InterruptedException ex) {
                        Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
            }
        }.start();
        return laOutFiles;
    }

    /**
     * Splitting Thread
     */
    private class SplitThread implements Runnable {

        private final int miPartIndex;
        private final long miSize;

        public SplitThread(int piPartIndex, long piSize) {
            miPartIndex = piPartIndex;
            miSize = piSize;
        }

        @Override
        public void run() {
            String lsDstPartFile = msDstDir + File.separator + msOutFileBaseName + "." + miPartIndex + ".part";
            System.out.println("FileSplitter: splitting " + lsDstPartFile);
            long liStartTime = System.currentTimeMillis();

            // output vars
            File loDstFile = new File(lsDstPartFile);
            RandomAccessFile loOutput = null;
            FileChannel loOutChannel = null;

            try {
                // Input Map
                MappedByteBuffer loInBuf = moInChannel.map(FileChannel.MapMode.READ_ONLY, miPartSize * miPartIndex, miSize);

                // Output Map
                if (!loDstFile.exists()) {
                    loDstFile.createNewFile();
                }
                loOutput = new RandomAccessFile(loDstFile, "rw");
                loOutChannel = loOutput.getChannel();
                MappedByteBuffer loOutBuf = loOutChannel.map(FileChannel.MapMode.READ_WRITE, 0, miSize);

                byte liByte;
                while (loInBuf.hasRemaining()) {
                    liByte = loInBuf.get();
                    loOutBuf.put(liByte);
                }
            } catch (IOException ex) {
                System.out.print("part_index:" + miPartIndex + " boom");
                Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
            } finally {
                // close output
                if (loOutChannel != null) {
                    try {
                        loOutChannel.close();
                    } catch (IOException ex) {
                        Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
                if (loOutput != null) {
                    try {
                        loOutput.close();
                    } catch (IOException ex) {
                        Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
            }

            System.out.println("Splitting Task " + lsDstPartFile + " Terminated. Spend " + (System.currentTimeMillis() - liStartTime) + " milsecs");
        }
    }
}

文件合并器：

package com.lonzh.fileutils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author Keven
 */
public class FileUnifier {

    private final String[] maSrcFiles;
    private final String msDstFile;

    private int miMaxThrPoolSize = 10;
    private ThreadPoolExecutor moThrPool;
    private RandomAccessFile moOutput;
    private FileChannel moOutputChannel;

    /**
     * Constructor
     *
     * @param paSrcFiles
     * @param psDstFile
     */
    public FileUnifier(String[] paSrcFiles, String psDstFile) {
        maSrcFiles = paSrcFiles;
        msDstFile = psDstFile;
    }

    /**
     * Constructor
     *
     * @param paSrcFiles
     * @param psDstFile
     * @param piMaxThrPool
     */
    public FileUnifier(String[] paSrcFiles, String psDstFile, int piMaxThrPool) {
        maSrcFiles = paSrcFiles;
        msDstFile = psDstFile;
        miMaxThrPoolSize = piMaxThrPool;
    }

    /**
     * Judge if the unifier is working
     *
     * @return
     */
    public boolean isBusy() {
        return moThrPool != null && !moThrPool.isTerminated();
    }

    /**
     * Start unifying
     *
     * @throws java.io.IOException
     */
    public void start() throws IOException {
        File loOutFile = new File(msDstFile);
        if (!loOutFile.exists()) {
            loOutFile.createNewFile();
        }
        moOutput = new RandomAccessFile(loOutFile, "rw");
        moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());

        // get total size
        long[][] laOffsetSizes = new long[maSrcFiles.length][2];
        long liOffset = 0;
        for (int i = 0; i < maSrcFiles.length; i++) {
            String lsSrcFile = maSrcFiles[i];
            File loFile = new File(lsSrcFile);
            if (!loFile.exists()) {
                throw new IOException("Src-File not exist:" + lsSrcFile);
            }
            laOffsetSizes[i][0] = liOffset;
            laOffsetSizes[i][1] = loFile.length();
            liOffset += loFile.length();
        }
        moOutput.setLength(liOffset);
        moOutputChannel = moOutput.getChannel();
        for (int i = 0; i < maSrcFiles.length; i++) {
            moThrPool.execute(new UnifyThread(maSrcFiles[i], laOffsetSizes[i][0], laOffsetSizes[i][1]));
        }
        moThrPool.shutdown();
        // Start monitor Thread
        new Thread() {
            @Override
            public void run() {
                while (true) {
                    if (moThrPool.isTerminated()) {
                        if (moOutputChannel != null) {
                            try {
                                moOutputChannel.close();
                            } catch (IOException ex) {
                                Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
                            }
                        }
                        if (moOutput != null) {
                            try {
                                moOutput.close();
                            } catch (IOException ex) {
                                Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
                            }
                        }
                        break;
                    }
                    try {
                        Thread.sleep(100);
                    } catch (InterruptedException ex) {
                        Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
            }
        }.start();
    }

    /**
     * Unifying thread
     */
    private class UnifyThread implements Runnable {

        private final String msSrcFile;
        private final long miOffset, miSize;

        public UnifyThread(String psSrcFile, long piOffset, long piSize) {
            msSrcFile = psSrcFile;
            miOffset = piOffset;
            miSize = piSize;
        }

        @Override
        public void run() {
            File loSrcFile = new File(msSrcFile);
            FileInputStream loInput = null;
            FileChannel loInChannel = null;
            try {
                System.out.println("FileUnifier: start unifying " + msSrcFile);
                long liStartTime = System.currentTimeMillis();
                loInput = new FileInputStream(loSrcFile);
                loInChannel = loInput.getChannel();
                MappedByteBuffer loInBuf = loInChannel.map(FileChannel.MapMode.READ_ONLY, 0, miSize);
                MappedByteBuffer loOutBuf = moOutputChannel.map(FileChannel.MapMode.READ_WRITE, miOffset, miSize);
                while (loInBuf.hasRemaining()) {
                    loOutBuf.put(loInBuf.get());
                }
                System.out.println("FileUnifier: unify " + msSrcFile + " complete. Spend " + (System.currentTimeMillis() - liStartTime) + "milsecs");
            } catch (FileNotFoundException ex) {
                Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
            } catch (IOException ex) {
                Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
            } finally {
                // Close input
                if (loInChannel != null) {
                    try {
                        loInChannel.close();
                    } catch (IOException ex) {
                        Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
                if (loInput != null) {
                    try {
                        loInput.close();
                    } catch (IOException ex) {
                        Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
            }
        }

    }
}

测试代码：

package test_java;

import com.lonzh.fileutils.FileSplitter;
import com.lonzh.fileutils.FileUnifier;
import java.io.File;
import java.io.IOException;

/**
 *
 * @author Keven
 */
public class Test_java {

    /**
     * @param args the command line arguments
     * @throws java.io.IOException
     * @throws java.lang.InterruptedException
     */
    public static void main(String[] args) throws IOException, InterruptedException {
        String lsSrcFile = "D:\\奔腾年代.rmvb";
        String lsDstDir = "D:\\parts";
        String lsDstFile = "D:\\parts\\dst.rmvb";
        File loDstFile = new File(lsDstDir);
        if (!loDstFile.exists()) {
            loDstFile.mkdir();
        }

        long liStartTime = System.currentTimeMillis();
        FileSplitter loSplitter = new FileSplitter(lsSrcFile, lsDstDir, "test_cut");
        String[] laPartFiles = loSplitter.start();
        while (loSplitter.isBusy()) {
            Thread.sleep(200);
        }
        FileUnifier loUnifier = new FileUnifier(laPartFiles, lsDstFile);
        loUnifier.start();
        while (loUnifier.isBusy()) {
            Thread.sleep(200);
        }
        System.out.println("done, spend " + (System.currentTimeMillis() - liStartTime) + " milsecs");
    }
}