Java实现分片上传与断点续传

最新推荐文章于 2024-05-08 11:28:38 发布

CodingBoyPP

最新推荐文章于 2024-05-08 11:28:38 发布

阅读量410

点赞数 10

分类专栏： Java 文章标签： java 分片上传断点续传

本文链接：https://blog.csdn.net/adminstritor/article/details/137981914

版权

Java 专栏收录该内容

9 篇文章 1 订阅

订阅专栏

1.写在前面

面对将大文件上传到服务器时，可能会遇到如下几个难点
1.整个大文件上传过程耗时比较长，而nginx一般会配置业务超时时间，很容易造成文件超时。
2.Nginx或着业务后端可能对上传文件大小进行了限制。
3.文件上传失败时，需要重新上传。

2.整体思路

1）分片上传实现
分片上传实现步骤

2）断点续传实现：根据前端传来分片总数创建BitMap, 每上传一个分片将对应位上置为1，通过计算0的位置来统计未上传的分片返回给前端。

3.代码实现

3.1 DTO对象

① 校验文件MD5请求DTO

@Data
public class FileCheckMD5RequestDTO {

    @ApiModelProperty("文件MD5")
    @NotEmpty
    private String md5;

    @ApiModelProperty("文件路径")
    @NotEmpty
    private String path;
}

② 分片上传请求DTO

@Data
@AllArgsConstructor
@NoArgsConstructor
@Accessors(chain = true)
@Builder
public class FileUploadRequestDTO {

    @ApiModelProperty(value = "任务ID, 传个uuid即可", notes = "传uuid即可")
    private String id;

    @ApiModelProperty("上传文件目录")
    @NotEmpty(message = "文件路径不能为空")
    private String path;

    @ApiModelProperty(value = "上传文件名称", notes = "这里不是指分片文件名")
    @NotEmpty(message = "文件名称不能为空")
    private String name;

    @ApiModelProperty("总分片数量")
    @NotNull(message = "分片数量不能为空")
    private Integer chunks;

    @ApiModelProperty(value = "当前为第几块分片", notes = "从0开始")
    @NotNull(message = "分片序号不能为空")
    private Integer chunk;

    @ApiModelProperty("按多大的文件粒度进行分片")
    @NotNull(message = "分片大小不能为空")
    private Long chunkSize;

    @ApiModelProperty("分片文件对象")
    private MultipartFile file;

    @ApiModelProperty(value = "文件MD5", notes = "用户上传的整个文件的md5")
    @NotEmpty(message = "文件MD5不能为空")
    private String md5;

    @ApiModelProperty("当前分片大小")
    private Long size = 0L;

    @ApiModelProperty(hidden = true)
    private byte[] fileBytes;
}

③ 响应DTO

@Builder
@Data
@AllArgsConstructor
@NoArgsConstructor
public class FileUploadDTO {

  @ApiModelProperty("上传文件地址")
  private String path;

  @ApiModelProperty("上传时间")
  private Long mtime;

  @ApiModelProperty("文件上传结果")
  private Boolean uploadComplete;

  @ApiModelProperty("上传结果码 200:文件已上传 404:文件没有上传过 206:文件只上传了一部分")
  private Integer code;

  @ApiModelProperty("分片信息")
  private Map<Integer,String> chunkMd5Info;

  @ApiModelProperty(value = "丢失的分片", notes = "断点续传")
  private List<Integer> missChunks;

  @ApiModelProperty("文件大小")
  private Long size;

  @ApiModelProperty("文件拓展名")
  private String fileExt;

  @ApiModelProperty("文件名")
  private String fileId;

}

3.2 接口层

public class FileController extends ApiController {

    @Autowired
    FileService fileService;

    @ApiOperation("文件上传")
    @PostMapping(value = "/upload")
    public R<FileUploadDTO> upload(@Validated FileUploadRequestDTO fileUploadRequestDTO, HttpServletRequest request) throws IOException {
        boolean isMultipart = ServletFileUpload.isMultipartContent(request);
        if (!isMultipart) {
            throw new MyException(ErrorCodeEnum.PARAMETER_ERROR);
        }

        FileUploadDTO fileUploadDTO;
        if (fileUploadRequestDTO.getChunk() != null && fileUploadRequestDTO.getChunks() > 0) {
            fileUploadDTO = fileService.sliceUpload(fileUploadRequestDTO);
        } else {
            fileUploadDTO = fileService.upload(fileUploadRequestDTO);
        }
        return R.ok(fileUploadDTO);
    }

    @ApiOperation("检查文件MD5")
    @PostMapping(value = "/checkFileMd5")
    public R<FileUploadDTO> checkFileMd5(@Validated @RequestBody FileCheckMD5RequestDTO dto) throws IOException {
        FileUploadRequestDTO param = new FileUploadRequestDTO().setPath(dto.getPath()).setMd5(dto.getMd5());
        FileUploadDTO fileUploadDTO = fileService.checkFileMd5(param);
        return R.ok(fileUploadDTO);
    }
}

3.3 具体实现

① 接口类

public interface FileService {

  /**
   * 单文件上传、不进行分片
   * @param fileUploadRequestDTO
   * @return
   * @throws IOException
   */
  FileUploadDTO upload(FileUploadRequestDTO fileUploadRequestDTO)throws IOException;

  /**
   * 分片上传
   * @param fileUploadRequestDTO
   * @return
   */
  FileUploadDTO sliceUpload(FileUploadRequestDTO fileUploadRequestDTO);

  /**
   * 校验文件md5值
   * 如分片文件未完全上传完毕，则返回缺省的分片md5
   * @param fileUploadRequestDTO
   * @return
   * @throws IOException
   */
  FileUploadDTO checkFileMd5(FileUploadRequestDTO fileUploadRequestDTO)throws IOException;
}

② 实现类

public class FileServiceImpl implements FileService {

    @Autowired
    private RedisUtil redisUtil;

    @Autowired
    @Qualifier("edge.uploadExecutor")
    private Executor uploadExecutor;

    private CompletionService<FileUploadDTO> completionService;

    @PostConstruct
    void init() {
        completionService = new ExecutorCompletionService<>(uploadExecutor,  new LinkedBlockingDeque<>(100));
    }

    @Override
    public FileUploadDTO upload(FileUploadRequestDTO param) throws IOException {
        if (Objects.isNull(param.getFile())) {
            throw new MyException("文件不能为空");
        }

        String md5 = com.ev.edge.utils.FileUtils.getFileMD5(param.getFile());
        param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
        param.setMd5(md5);
        String filePath = getPath(param);
        File targetFile = new File(filePath);
        if (!targetFile.exists()) {
            FileUtil.mkdir(targetFile);
        }

        String path = filePath + FileUtil.FILE_SEPARATOR + param.getFile().getOriginalFilename();
        FileUtil.writeBytes(param.getFile().getBytes(), path);
        redisUtil.set(FileConstant.FILE_UPLOAD_STATUS + md5, "true");
        return FileUploadDTO.builder().path(path).mtime(DateUtil.current()).uploadComplete(true).build();
    }

    @Override
    public FileUploadDTO sliceUpload(FileUploadRequestDTO fileUploadRequestDTO) {
        try {
            byte[] uploadFileBytes = IoUtil.readBytes(fileUploadRequestDTO.getFile().getInputStream());
            fileUploadRequestDTO.setFileBytes(uploadFileBytes);
            fileUploadRequestDTO.setFile(null);
            completionService.submit(new FileCallable(fileUploadRequestDTO));
            FileUploadDTO fileUploadDTO = completionService.take().get();
            return fileUploadDTO;
        } catch (Exception e) {
            log.error(e.getMessage(), e);
            throw new MyException(e.getMessage());
        }
    }

    @Override
    public FileUploadDTO checkFileMd5(FileUploadRequestDTO param) throws IOException {
        Object uploadProgressObj = redisUtil.get(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());;
        if (uploadProgressObj == null) {
            return FileUploadDTO.builder()
                    .code(FileCheckMd5Status.FILE_NO_UPLOAD.getValue()).build();
        }

        String processingStr = uploadProgressObj.toString();
        boolean processing = Boolean.parseBoolean(processingStr);
        String filePath = (String)redisUtil.get(FileConstant.FILE_MD5_KEY + param.getMd5());
        if (StrUtil.isEmpty(filePath)) {
            redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
            return FileUploadDTO.builder()
                    .code(FileCheckMd5Status.FILE_NO_UPLOAD.getValue()).build();
        }

        return fillFileUploadDTO(param, processing, filePath);
    }

    /**
     * 填充返回文件内容信息
     */
    private FileUploadDTO fillFileUploadDTO(FileUploadRequestDTO param, boolean processing,
                                            String value) throws IOException {
        // 如果上传已完成
        if (processing) {
            param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
            String path = getPath(param);
            return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOADED.getValue())
                    .path(path).build();
        }

        File confFile = new File(value);
        byte[] completeList = FileUtils.readFileToByteArray(confFile);
        List<Integer> missChunkList = new LinkedList<>();
        for (int i = 0; i < completeList.length; i++) {
            if (completeList[i] != Byte.MAX_VALUE) {
                missChunkList.add(i);
            }
        }

        // 如果丢失的分片为0, 则直接移动blob文件后视为上传成功
        if (CollUtil.isEmpty(missChunkList)) {
            File srcFile = new File(getPath(param) + "_tmp");
            File destFile = new File(param.getPath() + param.getName());
            FileUtil.move(srcFile, destFile, true);
            redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
            redisUtil.del(FileConstant.FILE_MD5_KEY + param.getMd5());
            return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOADED.getValue()).path(destFile.getPath()).build();
        }

        return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOAD_SOME.getValue())
                .missChunks(missChunkList).build();
    }

    public String getPath(FileUploadRequestDTO param) {
        String path = param.getPath() + FileUtil.FILE_SEPARATOR + param.getMd5();
        return path;
    }

③ 异步处理

public class FileCallable implements Callable<FileUploadDTO> {

  private FileUploadRequestDTO param;

  public FileCallable(FileUploadRequestDTO param) {
    this.param = param;
  }

  @Override
  public FileUploadDTO call() throws Exception {
    FileUploadDTO fileUploadDTO = new SliceUploadService().sliceUpload(param);
    return fileUploadDTO;
  }
}

④ 分片上传具体实现

public class SliceUploadService {

    @Value("${algo_server.storage.root}")
    String StorageRoot;

    /**
     * 文件分片大小, 默认4m
     */
    @Value("${upload.chunkSize:4}")
    private long defaultChunkSize;

    /**
     * 使用RandomAccessFile进行分片数据保存
     * @param param
     * @return
     */
    public boolean upload(FileUploadRequestDTO param) {
        RandomAccessFile accessTmpFile = null;
        try {
            String uploadDirPath = buildTmpFilePath(param);
            File tmpFile = this.createTmpFile(param);
            accessTmpFile = new RandomAccessFile(tmpFile, "rw");
            //这个必须与前端设定的值一致
            long chunkSize = Objects.isNull(param.getChunkSize()) ? defaultChunkSize * 1024 * 1024
                    : param.getChunkSize();
            long offset = chunkSize * param.getChunk();
            // 定位到该分片的偏移量
            accessTmpFile.seek(offset);
            // 写入该分片数据
            accessTmpFile.write(param.getFileBytes());
            // 同步进度
            boolean isOk = syncAndCheckUploadProgress(param, uploadDirPath);

            return isOk;
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            com.ev.edge.utils.FileUtils.close(accessTmpFile);
        }

        return false;
    }

    /**
     * 同步进度
     * 这里可能会存在多个线程对同一个文件进行访问，从而引发线程安全问题
     * @param param
     * @param uploadDirPath
     * @return
     */
    public boolean syncAndCheckUploadProgress(FileUploadRequestDTO param, String uploadDirPath) {
        RedisUtil redisUtil = SpringUtil.getBean(RedisUtil.class);
        try {
            while (true) {
                if (redisUtil.setSimpleLockWithLuaScript(FileConstant.FILE_SYNC + param.getMd5(), 3 * 60)) {
                    boolean isOk = checkAndSetUploadProgress(param, uploadDirPath);
                    return isOk;
                }
            }
        } finally {
            redisUtil.del(FileConstant.FILE_SYNC + param.getMd5());
        }
    }
    
    /**
     * 创建一个临时文件
     *
     * @param param
     * @return
     */
    protected File createTmpFile(FileUploadRequestDTO param) {
        param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
        String fileName = param.getName();
        String uploadDirPath = buildTmpFilePath(param);
        String tempFileName = fileName + "_tmp";
        File tmpDir = new File(uploadDirPath);
        File tmpFile = new File(uploadDirPath, tempFileName);
        if (!tmpDir.exists()) {
            tmpDir.mkdirs();
        }

        return tmpFile;
    }

    /**
     * 分片上传
     *
     * @param param
     * @return
     */
    @SneakyThrows
    public FileUploadDTO sliceUpload(FileUploadRequestDTO param) {
        // 文件分片上传
        boolean isOk = this.upload(param);

        // 上传成功则直接返回
        if (isOk) {
            log.debug("file {} upload completed!!!", param.getName());
            File tmpFile = this.createTmpFile(param);
            FileUploadDTO fileUploadDTO = this.saveAndFileUploadDTO(param, tmpFile);
            return fileUploadDTO;
        }

        String md5 = com.ev.edge.utils.FileUtils.getFileMD5Bak(param.getFileBytes());
        Map<Integer, String> map = new HashMap<>();
        map.put(param.getChunk(), md5);
        return FileUploadDTO.builder().chunkMd5Info(map).uploadComplete(false).build();
    }

    /**
     * 检查并修改文件上传进度
     */
    public boolean checkAndSetUploadProgress(FileUploadRequestDTO param, String uploadDirPath) {
        String fileName = param.getName();
        File confFile = new File(uploadDirPath, fileName + ".conf");
        byte isComplete = 0;
        RandomAccessFile accessConfFile = null;
        try {
            accessConfFile = new RandomAccessFile(confFile, "rw");
            // 把该分段标记为 true 表示完成
            log.debug("file:{} set part:{}  complete!", param.getName(), param.getChunk());

            // 创建conf文件文件长度为总分片数，每上传一个分块即向conf文件中写入一个127，那么没上传的位置就是默认0,已上传的就是Byte.MAX_VALUE 127
            accessConfFile.setLength(param.getChunks());
            accessConfFile.seek(param.getChunk());
            accessConfFile.write(Byte.MAX_VALUE);

            // completeList 检查是否全部完成,如果数组里是否全部都是127(全部分片都成功上传)
            byte[] completeList = FileUtils.readFileToByteArray(confFile);
            isComplete = Byte.MAX_VALUE;
            for (int i = 0; i < completeList.length && isComplete == Byte.MAX_VALUE; i++) {
                // 与运算, 如果有部分没有完成则 isComplete 不是 Byte.MAX_VALUE
                isComplete = (byte) (isComplete & completeList[i]);
                log.debug("file:{} check part: {} complete:{}", param.getName(), i, isComplete);
            }
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            com.ev.edge.utils.FileUtils.close(accessConfFile);
        }

        boolean isOk = setUploadProgress2Redis(param, uploadDirPath, fileName, confFile, isComplete);
        return isOk;
    }

    /**
     * 把上传进度信息存进redis
     */
    private boolean setUploadProgress2Redis(FileUploadRequestDTO param, String uploadDirPath,
                                            String fileName, File confFile, byte isComplete) {
        RedisUtil redisUtil = SpringUtil.getBean(RedisUtil.class);
        // 文件全部上传完成
        if (isComplete == Byte.MAX_VALUE) {
            log.debug("remove conf file: {}", confFile);
            FileUtil.del(confFile);
            redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
            redisUtil.del(FileConstant.FILE_MD5_KEY + param.getMd5());
            // 可以备份到临时目录用于秒传;
            return true;
        }

        // 断点续传10min后继续上传
        if (!redisUtil.hasKey(FileConstant.FILE_UPLOAD_STATUS + param.getMd5())) {
            redisUtil.set(FileConstant.FILE_UPLOAD_STATUS + param.getMd5(), "false");
            redisUtil.set(FileConstant.FILE_MD5_KEY + param.getMd5(), uploadDirPath + FileConstant.FILE_SEPARATORCHAR + fileName + ".conf", 10 * 60);
        }
        return false;
    }

    /**
     * 保存文件操作
     */
    public FileUploadDTO saveAndFileUploadDTO(FileUploadRequestDTO param, File tmpFile) {
        FileUploadDTO fileUploadDTO = null;
        int retryCount = 0;
        // 重试机制
        while (retryCount < 3) {
            try {
                fileUploadDTO = removeDestFile(tmpFile, param);
                if (fileUploadDTO.getUploadComplete()) {
                    log.info("file 【{}】 upload complete 【{}】!!", param.getName(), fileUploadDTO.getUploadComplete());
                }
                return fileUploadDTO;
            } catch (Exception e) {
                log.error(e.getMessage(), e);
                retryCount++;
            }
        }
        return fileUploadDTO;
    }

    /**
     * 移动文件
     *
     * @param srcFile   将要修改名字的文件
     * @param param 请求参数
     */
    private FileUploadDTO removeDestFile(File srcFile, FileUploadRequestDTO param) {
        //检查要重命名的文件是否存在，是否是文件
        FileUploadDTO fileUploadDTO = new FileUploadDTO();
        if (!srcFile.exists() || srcFile.isDirectory()) {
            log.info("File does not exist: {}", srcFile.getPath());
            fileUploadDTO.setUploadComplete(false);
            return fileUploadDTO;
        }

        String destFileName = param.getName();
        String ext = FileUtil.extName(destFileName);
        String filePath = param.getPath() + FileConstant.FILE_SEPARATORCHAR + destFileName;
        File destFile = new File(filePath);
        log.info("copy file from 【{}】 to 【{}】", srcFile.getPath(), destFile.getPath());
        FileUtil.move(srcFile, destFile, true);
        log.info("remove source file 【{}】", srcFile.getParentFile());
        FileUtil.del(srcFile.getParentFile());

        //修改文件名
        fileUploadDTO.setMtime(DateUtil.current());
        fileUploadDTO.setUploadComplete(true);
        fileUploadDTO.setPath(filePath);
        fileUploadDTO.setSize(destFile.length());
        fileUploadDTO.setFileExt(ext);
        fileUploadDTO.setFileId(destFileName);
        return fileUploadDTO;
    }

    /**
     * 构建临时文件存储路径:  指定文件模块路径/asdashjkhkjadas/
     * @param param
     * @return
     */
    public String buildTmpFilePath(FileUploadRequestDTO param) {
        String path = param.getPath() + FileUtil.FILE_SEPARATOR + param.getMd5();
        return path;
    }
}