大文件分片上传与下载、断点续传

大文件分片上传与下载

为什么要分片？

分片与并发结合，将一个大文件分割成多块，并发上传，极大地提高大文件的上传速度。
当网络问题导致传输错误时，只需要重传出错分片，而不是整个文件。另外分片传输能够更加实时的跟踪上传进度。

1、文件过大，超出服务端的请求大小限制；
2、请求时间过长，超时；
3、传输中断，必须重新上传导致前功尽弃；

做完了分片后，前端再发送一个请求给服务器，告诉它，上传完毕，把我们上传的几个分片合并成一个完整的文件。

@RestController
public class FileUploadDownloadController {

    private final static String UTF_8 = "utf-8";

    @RequestMapping("/upload")
    public void upload(HttpServletRequest request, HttpServletResponse response) throws Exception {

        //分片
        response.setCharacterEncoding(UTF_8);
        Integer schunk = null;
        Integer schunks = null;
        String name = null;
        String uploadPath = "D:\\fileItem";
        BufferedOutputStream os = null;
        try {
            DiskFileItemFactory factory = new DiskFileItemFactory();
            factory.setSizeThreshold(1024);
            factory.setRepository(new File(uploadPath));
            ServletFileUpload upload = new ServletFileUpload(factory);
            upload.setFileSizeMax(5l * 1024l * 1024l * 1024l);
            upload.setSizeMax(10l * 1024l * 1024l * 1024l);
            List<FileItem> items = upload.parseRequest(request);

            for (FileItem item : items) {
                if (item.isFormField()) {
                    if ("chunk".equals(item.getFieldName())) {
                        schunk = Integer.parseInt(item.getString(UTF_8));
                    }
                    if ("chunks".equals(item.getFieldName())) {
                        schunks = Integer.parseInt(item.getString(UTF_8));
                    }
                    if ("name".equals(item.getFieldName())) {
                        name = item.getString(UTF_8);
                    }
                }
            }
            for (FileItem item : items) {
                if (!item.isFormField()) {
                    String temFileName = name;
                    if (name != null) {
                        if (schunk != null) {
                            temFileName = schunk + "_" + name;
                        }
                        File temFile = new File(uploadPath, temFileName);
                        if (!temFile.exists()) {//断点续传
                            item.write(temFile);
                        }
                    }
                }
            }
            //文件合并
            if (schunk != null && schunk.intValue() == schunks.intValue() - 1) {
                File tempFile = new File(uploadPath, name);
                os = new BufferedOutputStream(new FileOutputStream(tempFile));

                for (int i = 0; i < schunks; i++) {
                    File file = new File(uploadPath, i + "_" + name);
                    while (!file.exists()) {
                        Thread.sleep(100);
                    }
                    byte[] bytes = FileUtils.readFileToByteArray(file);
                    os.write(bytes);
                    os.flush();
                    file.delete();
                }
                os.flush();
            }
            response.getWriter().write("上传成功" + name);
        } finally {
            try {
                if (os != null) {
                    os.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    @RequestMapping("/download")
    public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws Exception {
        File file = new File("C:\\Users\\admin\\nginx原理.mp4");
        response.setCharacterEncoding(UTF_8);
        InputStream is = null;
        OutputStream os = null;
        try {
            //分片下载   http  Range bytes=100-1000   bytes=100-
            long fSize = file.length();
            response.setContentType("application/x-download");
            String fileName = URLEncoder.encode(file.getName(), UTF_8);
            response.addHeader("Content-Disposition", "attachment;filename=" + fileName);
            response.setHeader("Accept-Range", "bytes");
            response.setHeader("fSize", String.valueOf(fSize));
            response.setHeader("fName", fileName);
            long pos = 0, last = fSize - 1, sum = 0;
            if (null != request.getHeader("Range")) {
                response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
                String numRange = request.getHeader("Range").replaceAll("bytes=", "");
                String[] strRange = numRange.split("-");
                if (strRange.length == 2) {
                    pos = Long.parseLong(strRange[0].trim());
                    last = Long.parseLong(strRange[1].trim());
                    if (last > fSize - 1) {
                        last = fSize - 1;
                    }
                } else {
                    pos = Long.parseLong(numRange.replaceAll("-", "").trim());
                }
            }
            long rangeLength = last - pos + 1;
            String contentRange = new StringBuffer("bytes ").append(pos).append("-").append(last).append("/").append(fSize).toString();
            response.setHeader("Content-Range", contentRange);
            response.setHeader("Content-Length", String.valueOf(rangeLength));

            os = new BufferedOutputStream(response.getOutputStream());
            is = new BufferedInputStream(new FileInputStream(file));
            is.skip(pos);
            byte[] buffer = new byte[1024];
            int lenght = 0;
            while (sum < rangeLength) {
                lenght = is.read(buffer, 0, ((rangeLength - sum) <= buffer.length ? ((int) (rangeLength - sum)) : buffer.length));
                sum = sum + lenght;
                os.write(buffer, 0, lenght);
            }
            System.out.println("下载完成");
        } finally {
            if (is != null) {
                is.close();
            }
            if (os != null) {
                os.close();
            }
        }
    }
}

1. @RestController
   public class DownloadClient {
   
       private final static long PER_PAGE = 1024L * 1024L * 1024L * 50L;
       private final static String DOWNLOAD_PATH = "D:\\fileItem";
       ExecutorService pool = Executors.newFixedThreadPool(10);
   
       @RequestMapping("/downloadFile")
       public String downloadFile() throws Exception {
           FileInfo fileInfo = download(0, 10, -1, null);
           //总分片数量
           if (null == fileInfo){
               throw new RuntimeException("下载异常！");
           }
           long pages = fileInfo.fSize / PER_PAGE;
           for (long i = 0; i <= pages; i++) {
               pool.submit(new Download(i * PER_PAGE, (i + 1) * PER_PAGE - 1, i, fileInfo.fName));
           }
           return "success";
       }
   
       class FileInfo {
           long fSize;
           String fName;
           public FileInfo(long fSize, String fName) {
               this.fSize = fSize;
               this.fName = fName;
           }
       }
   
       class Download implements Runnable {
           long start;
           long end;
           long page;
           String fName;
   
           public Download(long start, long end, long page, String fName) {
               this.start = start;
               this.end = end;
               this.page = page;
               this.fName = fName;
           }
   
           public void run() {
               try {
                   download(start, end, page, fName);
               } catch (Exception e) {
                   e.printStackTrace();
               }
           }
       }
   
       private FileInfo download(long start, long end, long page, String fName) throws Exception {
           File file = new File(DOWNLOAD_PATH, page + "-" + fName);
           if (file.exists()) {
               return null;
           }
           HttpClient client = HttpClients.createDefault();
           HttpGet httpGet = new HttpGet("http://127.0.0.1:8080/download");
           httpGet.setHeader("Range", "bytes=" + start + "-" + end);
   
           HttpResponse response = client.execute(httpGet);
           HttpEntity entity = response.getEntity();
           InputStream is = entity.getContent();
   
           String fSize = response.getFirstHeader("fSize").getValue();
           fName = URLDecoder.decode(response.getFirstHeader("fName").getValue(), "utf-8");
   
           FileOutputStream fis = new FileOutputStream(file);
           byte[] buffer = new byte[1024];
           int ch = 0;
           while ((ch = is.read(buffer)) != -1) {
               fis.write(buffer, 0, ch);
           }
           is.close();
           fis.flush();
           fis.close();
   
           if (end - Long.valueOf(fSize) >= 0) {//最后一个分片
               mergeFile(fName, page);
           }
           return new FileInfo(Long.valueOf(fSize), fName);
       }
   
       private void mergeFile(String fName, long page) throws Exception {
           File tempFile = new File(DOWNLOAD_PATH, fName);
           BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(tempFile));
   
           for (int i = 0; i <= page; i++) {
               File file = new File(DOWNLOAD_PATH, i + "-" + fName);
               while (!file.exists() || (i != page && file.length() < PER_PAGE)) {
                   Thread.sleep(100);
               }
               byte[] bytes = FileUtils.readFileToByteArray(file);
               os.write(bytes);
               os.flush();
               file.delete();
           }
           File file = new File(DOWNLOAD_PATH, -1 + "-null");
           file.delete();
           os.flush();
           os.close();
           //文件子节计算导致文件不完整
           //流未关闭
       }
   }

1、文件大小，文件名称
2、探测文件信息
3、多线程、分片下载
4、所有分片下载完毕，合并文件

public void cutFile(File bigFile,File destFile,int cutSize){

        FileInputStream inputStream = null;
        int size = 1024*1024; //1M
        try {
            if (!destFile.isDirectory()){ //如果保存分割文件的地址不是路径
                destFile.mkdir(); //创建路径
            }
            size = size * cutSize; //分割文件大小以M为单位
            int length = (int) bigFile.length(); //获取大文件大小（B为单位）
            int num = length / size; //计算分割成小文件的个数(每个小文件大小是M为单位)
            int yu = length % size; //除余的文件大小(M)

            String bigFilePath = bigFile.getAbsolutePath(); //获取大文件完整路径信息(包括文件名）
            int fileNew = bigFilePath.lastIndexOf("."); //获取文件后缀前的“."的索引
            String suffix = bigFilePath.substring(fileNew,bigFilePath.length()); //获取后缀，即文件类型

            inputStream = new FileInputStream(bigFile); //获取大文件的文件输入流
            File[] smallFile = new File[num+1]; //创建保存小文件的文件数组
            int begin = 0;
            for (int i =0;i< num;i++){
                smallFile[i] = new File(bigFile.getAbsolutePath()+"\\"+(i+1)+suffix+".tem"); //指定小文件的名字
                if (!smallFile[i].isFile()){
                    smallFile[i].createNewFile(); //创建该文件
                }
                FileOutputStream outputStream = new FileOutputStream(smallFile[i]); //创建小文件的文件输出流
                byte[] small = new byte[size];
                inputStream.read(small); //读取小文件字节
                outputStream.write(small); //向小文件中写入字节数据
                begin = begin + size;
                outputStream.close();
            }
            if (yu != 0){ ///除余的文件大小(M)部不为空
                smallFile[num] = new File(bigFile.getAbsolutePath()+"\\"+(num+1)+suffix+".tem");
                if (!smallFile[num].isFile()){
                    smallFile[num].createNewFile(); //创建文件
                }
                FileOutputStream outputStream = new FileOutputStream(smallFile[num]);
                byte[] bytes = new byte[yu];
                inputStream.read(bytes); //读取字节
                outputStream.write(bytes); //向文件写入数据
                outputStream.close();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

public void closeFile(File[] files,File closeDir,String hz){
        try {
            File closeFile = new File(closeDir.getAbsoluteFile()+"\\close"+hz); //指定合并后的文件名（包含路径）
            if (!closeFile.isFile()){
                closeFile.createNewFile(); //创建文件
            }

            FileOutputStream outputStream = new FileOutputStream(closeFile); //创建文件输出流
            for (int i=0;i<files.length;i++){
                FileInputStream inputStream = new FileInputStream(closeFile); //创建文件输入流
                int len = (int) files[i].length(); //获取单个子文件大小
                byte[] bytes = new byte[len];
                inputStream.read(bytes);
                outputStream.write(bytes);
                inputStream.close();
            }
            outputStream.close();
        }catch (Exception e){
            e.printStackTrace();
        }
    }

 下面的代码可以尝试下，超大文件合并

/**
* @param fileName 待分割的文件名 例：nginx.tar
* @return  key
*/
@GetMapping("/cutFile")
@ResponseBody
public String cutFile(String fileName){
   String key = String.valueOf(System.currentTimeMillis())+"-"+ fileName+"-key";
   stringRedisTemplate.boundValueOps(key).set("start");
   stringRedisTemplate.expire(key, 10, TimeUnit.MINUTES);

   CompletableFuture.runAsync(new Runnable() {
       @Override
       public void run() {
           List<String> fileNames = fileManageService.cutFile(fileName);
           if (CollectionUtils.isEmpty(fileNames)){
               stringRedisTemplate.boundValueOps(key).set("failed");
               stringRedisTemplate.expire(key, 1, TimeUnit.MINUTES);
           }

           if (!CollectionUtils.isEmpty(fileNames)){
               stringRedisTemplate.boundValueOps(key).set(JSONObject.toJSONString(fileNames));
               stringRedisTemplate.expire(key, 2, TimeUnit.MINUTES);
           }
       }
   });
   //返回key
   return key;
}

 分割文件

public List<String> sliceFile {
    @Value("${save_addr}")
    private String saveAddr;

   
    public List<String> cutFile(String fileName) {
        //待分片文件在主机上的路径
        String filePath = saveAddr + fileName;

        File file = new File(filePath);
        //分片文件的大小（字节）
        Long byteSize = 52428800L;
        List<String> fileNames = new CutFileUtil().cutFileBySize(filePath, byteSize, saveAddr);
        return fileNames;
    }
}

分割工具

/**
 * <功能简要> 

 * <切割文件工具>
 *
 * @Author heyanbo
 * @createTime 2020/6/7 23:31
 * @since
 */
public class CutFileUtil {

    /**
     * @param filePath 文件所在主机的路径 例：/home/gyt/nginx.tar
     * @param byteSize 拆分文件字节大小
     * @param saveAddr 拆分后的文件保存目录  /homt/gyt/
     * @return
     */
    public List<String> cutFileBySize(String filePath, Long byteSize, String saveAddr){
        List<String> fileNames = new ArrayList<>();
        File file = new File(filePath);
        //计算总共段数
        int count = (int) Math.ceil(file.length()/(double)byteSize);
        int countLen = (count +"").length();
        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(2,4,1,TimeUnit.SECONDS,new ArrayBlockingQueue<>(count * 2));
        //时间戳
        String timeStamp = String.valueOf(System.currentTimeMillis());

        for (int i = 0; i < count; i++) {
            //分段文件名
            String fileName = timeStamp + "-" + leftPad((i+1) +"", countLen, '0') + "-" +file.getName();
            threadPoolExecutor.execute(new SplitRunnable(byteSize.intValue(), fileName, file, i*byteSize, saveAddr));
            fileNames.add(fileName);
        }
        threadPoolExecutor.shutdown();
        while (true){
            if (threadPoolExecutor.isTerminated()){
                return fileNames;
            }
            try {
                Thread.sleep(1000);
            }catch (InterruptedException e){
                e.printStackTrace();
            }
        }
    }

    public static String leftPad(String str, int length, char ch){
        if (str.length() >= length){
            return str;
        }
        char[] chs = new char[length];
        Arrays.fill(chs, ch);
        char[] src = str.toCharArray();
        System.arraycopy(src, 0, chs, length - src.length, src.length);
        return new String(chs);
    }

    private class SplitRunnable implements Runnable{
        int byteSize;
        String fileName;
        File originFile;
        Long startPos;
        String currentWorkDir;

        public SplitRunnable(int byteSize, String fileName, File originFile, Long startPos, String currentWorkDir) {
            this.byteSize = byteSize;
            this.fileName = fileName;
            this.originFile = originFile;
            this.startPos = startPos;
            this.currentWorkDir = currentWorkDir;
        }

        public void run(){
            RandomAccessFile randomAccessFile = null;
            OutputStream outputStream = null;
            try {
                randomAccessFile = new RandomAccessFile(originFile, "r");
                byte[] b = new byte[byteSize];
                randomAccessFile.seek(startPos); //移动指针到每“段”开头
                int s = randomAccessFile.read(b);
                outputStream = new FileOutputStream(currentWorkDir+fileName);
                outputStream.write(b, 0 , s);
                outputStream.flush();
                b= null;
            }catch (IOException e){
                e.printStackTrace();
            }finally {
                if (outputStream !=null){
                    try {
                        outputStream.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }

                if (randomAccessFile !=null){
                    try {
                        randomAccessFile.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }
}

合并

	/**
     * @param cutFileName 任意一个分段文件名,例：1591604609899-1-redis.tar
     * @param chunks 分段总数
     * @return
     */
    @GetMapping("/merageFile")
    @ResponseBody
    public String merageFile(@RequestParam String cutFileName,
                             @RequestParam int chunks) throws IOException {
        return fileManageService.merageFile(cutFileName, chunks);
    }

public String merageFile(String cutFileName, int chunks) throws IOException {
        int indexOf = cutFileName.indexOf("-");
        String timeStream = cutFileName.substring(0, indexOf);
        //段数+文件名+后缀名
        String substring = cutFileName.substring(indexOf + 1, cutFileName.length());
        int indexOf1 = substring.indexOf("-");
        //文件名+后缀名
        String fileName = substring.substring(indexOf1+1, substring.length());
        File file = new File(saveAddr+fileName);
        if (file.exists()){
            file.delete();
            LOGGER.info("覆盖已经存在的文件");
        }
        BufferedOutputStream destOutputStream = new BufferedOutputStream(new FileOutputStream(saveAddr+fileName));
        for (int i = 1; i <= chunks ; i++) {
            //循环将每个分片的数据写入目标文件
            byte[] fileBuffer = new byte[1024];//文件读写缓存
            int readBytesLength = 0; //每次读取字节数
            File sourceFile = new File(saveAddr+timeStream+"-"+i+"-"+fileName);
            BufferedInputStream sourceInputStream = new BufferedInputStream(new FileInputStream(sourceFile));
            LOGGER.info("开始合并分段文件："+timeStream+"-"+i+"-"+fileName);
            while ((readBytesLength = sourceInputStream.read(fileBuffer))!=-1){
                destOutputStream.write(fileBuffer, 0 , readBytesLength);
            }
            sourceInputStream.close();
            LOGGER.info("合并分段文件完成："+timeStream+"-"+i+"-"+fileName);
            //分片合并后删除
            boolean delete = sourceFile.delete();
            if (delete){
                LOGGER.info(timeStream+"-"+i+"-"+fileName+"删除完成");
            }
        }
        destOutputStream.flush();
        destOutputStream.close();
        return fileName+"合并完成";
    }