大文件续传，文件分享

1. 最近各种文件分享平台，很多都要注册， 对于很多需要临时分享文件下的场景，不想被这种东西烦恼，于是借鉴网上代码，进行了一些修改， 写了一个文件分享项目， 该项目只是自用，数据库都没用， 就是简单的上传，下载， 删除也没有。如果有同样需求的同学可以看一下。

项目完整地址下载： cloudfile: 大文件断点续传， 文件分享，项目案例

下载后配置下路径和下载目录，正常运行即可

package com.zzk.cloudfile.demos.web;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.crypto.digest.DigestUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.zzk.cloudfile.demos.web.common.AjaxResult;
import com.zzk.cloudfile.demos.web.common.SliceBadException;
import com.zzk.cloudfile.demos.web.util.ExecutorUtils;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
import java.util.stream.Collectors;


@Controller
public class BasicController {

    private final String identification = "-slice-";
    private final String uploadslicedir = "uploads" + File.separator + "slice" + File.separator;//分片目录
    private final String uploaddir = "uploads" + File.separator + "real" + File.separator;//实际文件目录
//    private final String physicalPath = "D:\\cloudfile\\";
    private final String physicalPath = "/cloudfile/";
    private final String virtualPath = "http://10.123.123.125:8081/uploads/real/";


    // http://127.0.0.1:8080/hello?name=lisi
    @RequestMapping("/hello")
    @ResponseBody
    public String hello(@RequestParam(name = "name", defaultValue = "unknown user") String name) {
        return "Hello " + name;
    }


    //获取分片
    @GetMapping("/testing/{fileName}/{fileSliceSize}/{fileSize}")
    @ResponseBody
    public AjaxResult testing(@PathVariable String fileName, @PathVariable long fileSliceSize, @PathVariable long fileSize) {
        String dir = fileNameMd5Dir(fileName, fileSize);
        String absolutePath = physicalPath + uploadslicedir + dir;
        File file = FileUtil.mkdir(absolutePath);
        if (!file.exists()) {
            return AjaxResult.error();
        }

        List<File> filesAll = FileUtil.loopFiles(file.getAbsolutePath());
        if (filesAll.size() < 2) {
            //分片缺少 删除全部分片文件 ,从新上传
            FileUtil.clean(absolutePath);
            return AjaxResult.error();
        }

        //从小到大文件进行按照序号排序,和判断分片是否损坏
        List<String> collect = null;
        try {
            collect = fileSliceIsbadAndSort(file, fileSliceSize);
        } catch (Exception e) {
            if (e instanceof SliceBadException) {
                FileUtil.clean(absolutePath);
                return AjaxResult.error();
            }
        }
        //获取最后一个分片
        String fileSliceLatest = collect.get(collect.size() - 1);
        int code = fileId(fileSliceLatest);
        //服务器的分片总大小必须小于或者等于文件的总大小
        if ((code * fileSliceSize) <= fileSize) {
            JSONObject obj = JSONUtil.createObj();
            obj.set("code", String.valueOf(code));
            obj.set("fileSliceLatest", fileSliceLatest);
            obj.set("progressNow", collect.size());
            return AjaxResult.success(obj);
        } else {
            //分片异常 ,删除全部分片文件,从新上传
            FileUtil.clean(absolutePath);
            return AjaxResult.error();
        }
    }

    @PostMapping(value = "/uploads")
    @ResponseBody
    public AjaxResult uploads(HttpServletRequest request, @RequestParam("part") MultipartFile part) throws IOException {
        String fileSliceName = request.getParameter("fileSliceName");
        long fileSize = Long.parseLong(request.getParameter("fileSize")); //文件大小
        String dir = fileSliceMd5Dir(fileSliceName, fileSize);
        File file = FileUtil.mkdir(physicalPath + uploadslicedir + dir + File.separator + fileSliceName);
        part.transferTo(file);
        int i = fileId(file.getName());
//        try {
//            //模拟耗时操作，要不前端显示太快,正式时删除
//            Thread.sleep(RandomUtil.randomInt(2000, 5000, true, true));
//        } catch (InterruptedException e) {
//            e.printStackTrace();
//        }
        return AjaxResult.success(i);
    }

    // 合并分片
    @GetMapping(value = "/merge-file-slice/{fileSliceName}/{fileSliceSize}/{fileSize}")
    @ResponseBody
    public AjaxResult mergeFileSlice(@PathVariable String fileSliceName, @PathVariable long fileSliceSize, @PathVariable long fileSize) throws Exception {
        int l = (int) Math.ceil((double) fileSize / fileSliceSize); //有多少个分片
        String dir = fileSliceMd5Dir(fileSliceName, fileSize); //分片所在的目录
        String absolutePath = physicalPath + uploadslicedir + dir;
        File file = FileUtil.mkdir(absolutePath);
        String realFileName;
        String realFileNamePath;

        if (file.exists()) {
            List<String> filesAll = FileUtil.listFileNames(absolutePath);


            //阻塞循环判断是否还在上传  ,解决前端进行ajax异步上传的问题
            int beforeSize = filesAll.size();
            while (true) {
                if (filesAll.size() == l) {
                    break;
                }
                Thread.sleep(1000);
                //之前分片数量和现在分片数据之差,如果大于1那么就在上传,那么继续
                filesAll = FileUtil.listFileNames(absolutePath);
                if (filesAll.size() - beforeSize >= 1) {
                    beforeSize = filesAll.size();
                    //继续检测
                    continue;
                }
                //如果是之前分片和现在的分片相等的,那么在阻塞2秒后检测是否发生变化,如果还没变化那么上传全部完成,可以进行合并了
                //当然这不是绝对的,只能解决网络短暂的波动,因为有可能发生断网很长时间,网络恢复后文件恢复上传, 这个问题是避免不了的,所以我们在下面的代码进行数量的效验
                // 因为我们不可能一直等着他网好,所以如果1~3秒内没有上传新的内容,那么我们默认判定上传完毕
                if (beforeSize == filesAll.size()) {
                    Thread.sleep(2000);
                    filesAll = FileUtil.listFileNames(absolutePath);
                    if (beforeSize == filesAll.size()) {
                        break;
                    }
                }
            }
            //分片数量效验
            if (filesAll.size() != l) {
                //分片缺少 ,删除全部分片文件,从新上传
                FileUtil.clean(absolutePath);
                return AjaxResult.error();
            }
            //获取实际的文件名称,组装路径
            realFileName = realFileName(fileSliceName);
            realFileNamePath = physicalPath + uploaddir + realFileName;

            //从小到大文件进行按照序号排序 ,和检查分片文件是否有问题
            List<String> collect = fileSliceIsbadAndSort(file, fileSliceSize);
            int fileSliceCount = collect.size();
            FileUtil.touch(realFileNamePath);

            List<Future<?>> futures = new ArrayList<>();
            for (int i = 0; i < fileSliceCount; i++) {
                int finalI = i;
                Future<?> future = ExecutorUtils.createFuture(() -> {
                    String fileNameTemp = collect.get(finalI);
                    long fileTempSize = new File(absolutePath + File.separator+fileNameTemp).length();
                    byte[] bytes = new byte[(int) fileTempSize];
                    try (RandomAccessFile r = new RandomAccessFile(absolutePath + File.separator +fileNameTemp, "r")) {
                        r.read(bytes, 0, (int) fileTempSize);
                    } catch (IOException e) {
                        e.printStackTrace();
                    }

                    try (RandomAccessFile w = new RandomAccessFile(realFileNamePath, "rw")) {
                        //当前文件写入的位置
                        w.seek(finalI * fileSliceSize);
                        w.write(bytes);
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    return 1;
                });
                futures.add(future);
            }
            //阻塞到全部线程执行完毕后
            ExecutorUtils.waitComplete(futures);
            //删除全部分片文件
            FileUtil.clean(absolutePath);
        } else {
            //没有这个分片相关的的目录
            return AjaxResult.error();
        }
        return AjaxResult.success(virtualPath + realFileName);
    }


    //获取分片文件的目录
    private String fileSliceMd5Dir(String fileSliceName, long fileSize) {
        int i = fileSliceName.indexOf(identification);
        String substring = fileSliceName.substring(0, i);
        String dir = DigestUtil.md5Hex(substring + fileSize);
        return dir;
    }

    //通过文件名称获取文件目录
    private String fileNameMd5Dir(String fileName, long fileSize) {
        return DigestUtil.md5Hex(fileName + fileSize);
    }

    //获取分片的实际文件名
    private String realFileName(String fileSliceName) {
        int i = fileSliceName.indexOf(identification);
        String substring = fileSliceName.substring(0, i);
        return substring;

    }

    //获取文件序号
    private int fileId(String fileSliceName) {
        int i = fileSliceName.indexOf(identification) + identification.length();
        String fileId = fileSliceName.substring(i);
        return Integer.parseInt(fileId);
    }

    //判断是否损坏
    private List<String> fileSliceIsbadAndSort(File file, long fileSliceSize) throws Exception {
        String absolutePath = file.getAbsolutePath();
        List<String> filesAll = FileUtil.listFileNames(absolutePath);
        if (filesAll.size() < 1) {
            //分片缺少,删除全部分片文件 ,从新上传
            FileUtil.clean(absolutePath);
            throw new SliceBadException();
        }
        //从小到大文件进行按照序号排序
        List<String> collect = filesAll.stream().sorted((a, b) -> fileId(a) - fileId(b)).collect(Collectors.toList());
        //判断文件是否损坏,将文件排序后,进行前后序号相差大于1那么就代表少分片了
        for (int i = 0; i < collect.size() - 1; i++) {
            //检测分片的连续度
            if (fileId(collect.get(i)) - fileId(collect.get(i + 1)) != -1) {
                //分片损坏 删除全部分片文件 ,从新上传
                FileUtil.clean(absolutePath);
                throw new SliceBadException();
            }
            //检测分片的完整度
            if (new File(absolutePath + File.separator + collect.get(i)).length() != fileSliceSize) {
                //分片损坏 删除全部分片文件 ,从新上传
                FileUtil.clean(absolutePath);
                throw new SliceBadException();
            }
        }
        return collect;
    }


}

//大文件分片上传,比如10G的压缩包,或者视频等,这些文件太大了  (需要后端配合进行)
class FileSliceUpload {

    constructor(testingUrl, uploadUrl, margeUrl, fileSelect) {
        this.testingUrl = testingUrl; // 检测文件上传的url
        this.uploadUrl = uploadUrl;//文件上传接口
        this.margeUrl = margeUrl; // 合并文件接口
        this.fileSelect = fileSelect;
        this.fileObj = null;
        this.totalsize = null;
        this.blockSize = 1024 * 1024; //每次上传多少字节1mb(最佳)
        this.sta = 0; //起始位置
        this.end = this.sta + this.blockSize; //结束位置
        this.count = 0; //分片个数
        this.barId = "bar"; //进度条id
        this.progressId = "progress";//进度数值ID
        this.progressNow = 0;
        this.progressTotal = 0;
        this.fileSliceName = ""; //分片文件名称
        this.fileName = "";
        this.uploadFileInterval = null;  //上传文件定时器

    }

    /**
     *  样式可以进行修改
     * @param {*} progressId   需要将进度条添加到那个元素下面
     */
    addProgress(progressSelect) {
        let bar = document.createElement("div")
        bar.setAttribute("id", this.barId);
        let num = document.createElement("div")
        num.setAttribute("id", this.progressId);
        num.innerText = "0%"
        bar.appendChild(num);
        document.querySelector(progressSelect).appendChild(bar)

    }

    //续传  在上传前先去服务器检测之前是否有上传过这个文件,如果还有返回上传的的分片,那么进行续传
    // 将当前服务器上传的最后一个分片会从新上传, 避免因为网络的原因导致分片损坏
    sequelFile() {
        if (this.fileName) {
            var xhr = new XMLHttpRequest();
            //同步
            var url = this.testingUrl + "/" + this.fileName + "/" + this.blockSize + "/" + this.totalsize;
            xhr.open('GET', encodeURI(url), false);
            xhr.send();
            if (xhr.readyState === 4 && xhr.status === 200) {
                let ret = JSON.parse(xhr.response)
                if (ret.code == 0) {
                    let data = ret.data
                    this.count = data.code;
                    this.fileSliceName = data.fileSliceName
                    //计算起始位置和结束位置
                    this.sta = this.blockSize * this.count
                    //计算结束位置
                    this.end = this.sta + this.blockSize
                    this.progressNow = data.progressNow
                } else {
                    this.sta = 0; //从头开始
                    this.end = this.sta + this.blockSize;
                    this.count = 0; //分片个数
                    this.progressNow = 0;
                }
            }
        }
    }

    stopUploadFile() {
        console.log("stop!!!")
        clearInterval(this.uploadFileInterval)
    }

    // 文件上传(单文件)
    startUploadFile() {
        this.resetProgress();

        this.fileObj = document.querySelector(this.fileSelect).files[0];
        this.totalsize = this.fileObj.size;
        this.fileName = this.fileObj.name;
        this.progressTotal = Math.ceil(this.totalsize / this.blockSize)
        console.log("文件总大小：" + this.totalsize + ", 需要分片个数： " + this.progressTotal )

        //查询是否存在之前上传过此文件,然后继续
        this.sequelFile()
        let ref = this; //拿到当前对象的引用,因为是在异步中使用this就是他本身而不是class
        this.uploadFileInterval = setInterval(function () {
            if (ref.sta > ref.totalsize) {
                //上传完毕后结束定时器
                clearInterval(ref.uploadFileInterval)
                //发送合并请求
                ref.margeUploadFile()
                console.log("stop" + ref.sta);
                return;
            }
            ;
            //分片名称
            ref.fileSliceName = ref.fileName + "-slice-" + ref.count++
            //分割文件 ,
            var blob1 = ref.fileObj.slice(ref.sta, ref.end);
            console.log("start: "+ ref.sta + ", end : "+ ref.end)
            var fd = new FormData();
            fd.append('part', blob1);
            fd.append('fileSliceName', ref.fileSliceName);
            fd.append('fileSize', ref.totalsize);
            var xhr = new XMLHttpRequest();
            xhr.open('POST', ref.uploadUrl, true);
            xhr.send(fd); //异步发送文件,不管是否成功, 会定期检测

            xhr.onreadystatechange = function () {
                if (xhr.readyState === 4 && xhr.status === 200) {
                    let ret = JSON.parse(xhr.response)
                    if (ret.code == 0) {
                        console.log(ret.data);
                        ref.progressNow++;
                        ref.calculateProgress();
                    }
                }
            }
            //起始位置等于上次上传的结束位置
            ref.sta = ref.end;
            //结束位置等于上次上传的结束位置+每次上传的字节
            ref.end = ref.sta + ref.blockSize;
        }, 100)

    }

    margeUploadFile() {
        console.log("检测上传的文件完整性..........");
        var xhr = new XMLHttpRequest();
        //文件分片的名称/分片大小/总大小
        var url = this.margeUrl + "/" + this.fileSliceName + "/" + this.blockSize + "/" + this.totalsize;
        xhr.open('GET', encodeURI(url), true);
        xhr.send(); //发送请求
        xhr.onreadystatechange = function () {
            if (xhr.readyState === 4 && xhr.status === 200) {
                let ret = JSON.parse(xhr.response)
                if (ret.code == 0) {
                    console.log("文件上传完毕");
                    let address = document.getElementById("address");
                    address.value = ret.msg;
                    address.style.display = "block"
                } else {
                    console.log("上传完毕但是文件上传过程中出现了异常", ret);
                }
            }
        }

    }

    calculateProgress() {
        let bar = document.getElementById(this.barId)
        let progressEl = document.getElementById(this.progressId)
        let percent = Math.ceil((this.progressNow / this.progressTotal) * 100)
        if (percent > 100) {
            percent = 100
        }
        bar.style.width = percent + '%';
        bar.style.backgroundColor = 'green';
        progressEl.innerHTML = percent + '%'
    }

    resetProgress(){
        let bar = document.getElementById(this.barId)
        let progressEl = document.getElementById(this.progressId)
        bar.style.width = 0 + '%';
        bar.style.backgroundColor = 'green';
        progressEl.innerHTML = 0 + '%'
    }

}