1.写在前面
面对将大文件上传到服务器时,可能会遇到如下几个难点
1.整个大文件上传过程耗时比较长,而nginx一般会配置业务超时时间,很容易造成文件超时。
2.Nginx或着业务后端可能对上传文件大小进行了限制。
3.文件上传失败时,需要重新上传。
2.整体思路
1)分片上传实现
2)断点续传实现:根据前端传来分片总数创建BitMap, 每上传一个分片将对应位上置为1,通过计算0的位置来统计未上传的分片返回给前端。
3.代码实现
3.1 DTO对象
① 校验文件MD5请求DTO
@Data
public class FileCheckMD5RequestDTO {
@ApiModelProperty("文件MD5")
@NotEmpty
private String md5;
@ApiModelProperty("文件路径")
@NotEmpty
private String path;
}
② 分片上传请求DTO
@Data
@AllArgsConstructor
@NoArgsConstructor
@Accessors(chain = true)
@Builder
public class FileUploadRequestDTO {
@ApiModelProperty(value = "任务ID, 传个uuid即可", notes = "传uuid即可")
private String id;
@ApiModelProperty("上传文件目录")
@NotEmpty(message = "文件路径不能为空")
private String path;
@ApiModelProperty(value = "上传文件名称", notes = "这里不是指分片文件名")
@NotEmpty(message = "文件名称不能为空")
private String name;
@ApiModelProperty("总分片数量")
@NotNull(message = "分片数量不能为空")
private Integer chunks;
@ApiModelProperty(value = "当前为第几块分片", notes = "从0开始")
@NotNull(message = "分片序号不能为空")
private Integer chunk;
@ApiModelProperty("按多大的文件粒度进行分片")
@NotNull(message = "分片大小不能为空")
private Long chunkSize;
@ApiModelProperty("分片文件对象")
private MultipartFile file;
@ApiModelProperty(value = "文件MD5", notes = "用户上传的整个文件的md5")
@NotEmpty(message = "文件MD5不能为空")
private String md5;
@ApiModelProperty("当前分片大小")
private Long size = 0L;
@ApiModelProperty(hidden = true)
private byte[] fileBytes;
}
③ 响应DTO
@Builder
@Data
@AllArgsConstructor
@NoArgsConstructor
public class FileUploadDTO {
@ApiModelProperty("上传文件地址")
private String path;
@ApiModelProperty("上传时间")
private Long mtime;
@ApiModelProperty("文件上传结果")
private Boolean uploadComplete;
@ApiModelProperty("上传结果码 200:文件已上传 404:文件没有上传过 206:文件只上传了一部分")
private Integer code;
@ApiModelProperty("分片信息")
private Map<Integer,String> chunkMd5Info;
@ApiModelProperty(value = "丢失的分片", notes = "断点续传")
private List<Integer> missChunks;
@ApiModelProperty("文件大小")
private Long size;
@ApiModelProperty("文件拓展名")
private String fileExt;
@ApiModelProperty("文件名")
private String fileId;
}
3.2 接口层
public class FileController extends ApiController {
@Autowired
FileService fileService;
@ApiOperation("文件上传")
@PostMapping(value = "/upload")
public R<FileUploadDTO> upload(@Validated FileUploadRequestDTO fileUploadRequestDTO, HttpServletRequest request) throws IOException {
boolean isMultipart = ServletFileUpload.isMultipartContent(request);
if (!isMultipart) {
throw new MyException(ErrorCodeEnum.PARAMETER_ERROR);
}
FileUploadDTO fileUploadDTO;
if (fileUploadRequestDTO.getChunk() != null && fileUploadRequestDTO.getChunks() > 0) {
fileUploadDTO = fileService.sliceUpload(fileUploadRequestDTO);
} else {
fileUploadDTO = fileService.upload(fileUploadRequestDTO);
}
return R.ok(fileUploadDTO);
}
@ApiOperation("检查文件MD5")
@PostMapping(value = "/checkFileMd5")
public R<FileUploadDTO> checkFileMd5(@Validated @RequestBody FileCheckMD5RequestDTO dto) throws IOException {
FileUploadRequestDTO param = new FileUploadRequestDTO().setPath(dto.getPath()).setMd5(dto.getMd5());
FileUploadDTO fileUploadDTO = fileService.checkFileMd5(param);
return R.ok(fileUploadDTO);
}
}
3.3 具体实现
① 接口类
public interface FileService {
/**
* 单文件上传、不进行分片
* @param fileUploadRequestDTO
* @return
* @throws IOException
*/
FileUploadDTO upload(FileUploadRequestDTO fileUploadRequestDTO)throws IOException;
/**
* 分片上传
* @param fileUploadRequestDTO
* @return
*/
FileUploadDTO sliceUpload(FileUploadRequestDTO fileUploadRequestDTO);
/**
* 校验文件md5值
* 如分片文件未完全上传完毕,则返回缺省的分片md5
* @param fileUploadRequestDTO
* @return
* @throws IOException
*/
FileUploadDTO checkFileMd5(FileUploadRequestDTO fileUploadRequestDTO)throws IOException;
}
② 实现类
public class FileServiceImpl implements FileService {
@Autowired
private RedisUtil redisUtil;
@Autowired
@Qualifier("edge.uploadExecutor")
private Executor uploadExecutor;
private CompletionService<FileUploadDTO> completionService;
@PostConstruct
void init() {
completionService = new ExecutorCompletionService<>(uploadExecutor, new LinkedBlockingDeque<>(100));
}
@Override
public FileUploadDTO upload(FileUploadRequestDTO param) throws IOException {
if (Objects.isNull(param.getFile())) {
throw new MyException("文件不能为空");
}
String md5 = com.ev.edge.utils.FileUtils.getFileMD5(param.getFile());
param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
param.setMd5(md5);
String filePath = getPath(param);
File targetFile = new File(filePath);
if (!targetFile.exists()) {
FileUtil.mkdir(targetFile);
}
String path = filePath + FileUtil.FILE_SEPARATOR + param.getFile().getOriginalFilename();
FileUtil.writeBytes(param.getFile().getBytes(), path);
redisUtil.set(FileConstant.FILE_UPLOAD_STATUS + md5, "true");
return FileUploadDTO.builder().path(path).mtime(DateUtil.current()).uploadComplete(true).build();
}
@Override
public FileUploadDTO sliceUpload(FileUploadRequestDTO fileUploadRequestDTO) {
try {
byte[] uploadFileBytes = IoUtil.readBytes(fileUploadRequestDTO.getFile().getInputStream());
fileUploadRequestDTO.setFileBytes(uploadFileBytes);
fileUploadRequestDTO.setFile(null);
completionService.submit(new FileCallable(fileUploadRequestDTO));
FileUploadDTO fileUploadDTO = completionService.take().get();
return fileUploadDTO;
} catch (Exception e) {
log.error(e.getMessage(), e);
throw new MyException(e.getMessage());
}
}
@Override
public FileUploadDTO checkFileMd5(FileUploadRequestDTO param) throws IOException {
Object uploadProgressObj = redisUtil.get(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());;
if (uploadProgressObj == null) {
return FileUploadDTO.builder()
.code(FileCheckMd5Status.FILE_NO_UPLOAD.getValue()).build();
}
String processingStr = uploadProgressObj.toString();
boolean processing = Boolean.parseBoolean(processingStr);
String filePath = (String)redisUtil.get(FileConstant.FILE_MD5_KEY + param.getMd5());
if (StrUtil.isEmpty(filePath)) {
redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
return FileUploadDTO.builder()
.code(FileCheckMd5Status.FILE_NO_UPLOAD.getValue()).build();
}
return fillFileUploadDTO(param, processing, filePath);
}
/**
* 填充返回文件内容信息
*/
private FileUploadDTO fillFileUploadDTO(FileUploadRequestDTO param, boolean processing,
String value) throws IOException {
// 如果上传已完成
if (processing) {
param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
String path = getPath(param);
return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOADED.getValue())
.path(path).build();
}
File confFile = new File(value);
byte[] completeList = FileUtils.readFileToByteArray(confFile);
List<Integer> missChunkList = new LinkedList<>();
for (int i = 0; i < completeList.length; i++) {
if (completeList[i] != Byte.MAX_VALUE) {
missChunkList.add(i);
}
}
// 如果丢失的分片为0, 则直接移动blob文件后视为上传成功
if (CollUtil.isEmpty(missChunkList)) {
File srcFile = new File(getPath(param) + "_tmp");
File destFile = new File(param.getPath() + param.getName());
FileUtil.move(srcFile, destFile, true);
redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
redisUtil.del(FileConstant.FILE_MD5_KEY + param.getMd5());
return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOADED.getValue()).path(destFile.getPath()).build();
}
return FileUploadDTO.builder().code(FileCheckMd5Status.FILE_UPLOAD_SOME.getValue())
.missChunks(missChunkList).build();
}
public String getPath(FileUploadRequestDTO param) {
String path = param.getPath() + FileUtil.FILE_SEPARATOR + param.getMd5();
return path;
}
③ 异步处理
public class FileCallable implements Callable<FileUploadDTO> {
private FileUploadRequestDTO param;
public FileCallable(FileUploadRequestDTO param) {
this.param = param;
}
@Override
public FileUploadDTO call() throws Exception {
FileUploadDTO fileUploadDTO = new SliceUploadService().sliceUpload(param);
return fileUploadDTO;
}
}
④ 分片上传具体实现
public class SliceUploadService {
@Value("${algo_server.storage.root}")
String StorageRoot;
/**
* 文件分片大小, 默认4m
*/
@Value("${upload.chunkSize:4}")
private long defaultChunkSize;
/**
* 使用RandomAccessFile进行分片数据保存
* @param param
* @return
*/
public boolean upload(FileUploadRequestDTO param) {
RandomAccessFile accessTmpFile = null;
try {
String uploadDirPath = buildTmpFilePath(param);
File tmpFile = this.createTmpFile(param);
accessTmpFile = new RandomAccessFile(tmpFile, "rw");
//这个必须与前端设定的值一致
long chunkSize = Objects.isNull(param.getChunkSize()) ? defaultChunkSize * 1024 * 1024
: param.getChunkSize();
long offset = chunkSize * param.getChunk();
// 定位到该分片的偏移量
accessTmpFile.seek(offset);
// 写入该分片数据
accessTmpFile.write(param.getFileBytes());
// 同步进度
boolean isOk = syncAndCheckUploadProgress(param, uploadDirPath);
return isOk;
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
com.ev.edge.utils.FileUtils.close(accessTmpFile);
}
return false;
}
/**
* 同步进度
* 这里可能会存在多个线程对同一个文件进行访问,从而引发线程安全问题
* @param param
* @param uploadDirPath
* @return
*/
public boolean syncAndCheckUploadProgress(FileUploadRequestDTO param, String uploadDirPath) {
RedisUtil redisUtil = SpringUtil.getBean(RedisUtil.class);
try {
while (true) {
if (redisUtil.setSimpleLockWithLuaScript(FileConstant.FILE_SYNC + param.getMd5(), 3 * 60)) {
boolean isOk = checkAndSetUploadProgress(param, uploadDirPath);
return isOk;
}
}
} finally {
redisUtil.del(FileConstant.FILE_SYNC + param.getMd5());
}
}
/**
* 创建一个临时文件
*
* @param param
* @return
*/
protected File createTmpFile(FileUploadRequestDTO param) {
param.setPath(com.ev.edge.utils.FileUtils.withoutTailDiagonal(param.getPath()));
String fileName = param.getName();
String uploadDirPath = buildTmpFilePath(param);
String tempFileName = fileName + "_tmp";
File tmpDir = new File(uploadDirPath);
File tmpFile = new File(uploadDirPath, tempFileName);
if (!tmpDir.exists()) {
tmpDir.mkdirs();
}
return tmpFile;
}
/**
* 分片上传
*
* @param param
* @return
*/
@SneakyThrows
public FileUploadDTO sliceUpload(FileUploadRequestDTO param) {
// 文件分片上传
boolean isOk = this.upload(param);
// 上传成功则直接返回
if (isOk) {
log.debug("file {} upload completed!!!", param.getName());
File tmpFile = this.createTmpFile(param);
FileUploadDTO fileUploadDTO = this.saveAndFileUploadDTO(param, tmpFile);
return fileUploadDTO;
}
String md5 = com.ev.edge.utils.FileUtils.getFileMD5Bak(param.getFileBytes());
Map<Integer, String> map = new HashMap<>();
map.put(param.getChunk(), md5);
return FileUploadDTO.builder().chunkMd5Info(map).uploadComplete(false).build();
}
/**
* 检查并修改文件上传进度
*/
public boolean checkAndSetUploadProgress(FileUploadRequestDTO param, String uploadDirPath) {
String fileName = param.getName();
File confFile = new File(uploadDirPath, fileName + ".conf");
byte isComplete = 0;
RandomAccessFile accessConfFile = null;
try {
accessConfFile = new RandomAccessFile(confFile, "rw");
// 把该分段标记为 true 表示完成
log.debug("file:{} set part:{} complete!", param.getName(), param.getChunk());
// 创建conf文件文件长度为总分片数,每上传一个分块即向conf文件中写入一个127,那么没上传的位置就是默认0,已上传的就是Byte.MAX_VALUE 127
accessConfFile.setLength(param.getChunks());
accessConfFile.seek(param.getChunk());
accessConfFile.write(Byte.MAX_VALUE);
// completeList 检查是否全部完成,如果数组里是否全部都是127(全部分片都成功上传)
byte[] completeList = FileUtils.readFileToByteArray(confFile);
isComplete = Byte.MAX_VALUE;
for (int i = 0; i < completeList.length && isComplete == Byte.MAX_VALUE; i++) {
// 与运算, 如果有部分没有完成则 isComplete 不是 Byte.MAX_VALUE
isComplete = (byte) (isComplete & completeList[i]);
log.debug("file:{} check part: {} complete:{}", param.getName(), i, isComplete);
}
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
com.ev.edge.utils.FileUtils.close(accessConfFile);
}
boolean isOk = setUploadProgress2Redis(param, uploadDirPath, fileName, confFile, isComplete);
return isOk;
}
/**
* 把上传进度信息存进redis
*/
private boolean setUploadProgress2Redis(FileUploadRequestDTO param, String uploadDirPath,
String fileName, File confFile, byte isComplete) {
RedisUtil redisUtil = SpringUtil.getBean(RedisUtil.class);
// 文件全部上传完成
if (isComplete == Byte.MAX_VALUE) {
log.debug("remove conf file: {}", confFile);
FileUtil.del(confFile);
redisUtil.del(FileConstant.FILE_UPLOAD_STATUS + param.getMd5());
redisUtil.del(FileConstant.FILE_MD5_KEY + param.getMd5());
// 可以备份到临时目录用于秒传;
return true;
}
// 断点续传10min后继续上传
if (!redisUtil.hasKey(FileConstant.FILE_UPLOAD_STATUS + param.getMd5())) {
redisUtil.set(FileConstant.FILE_UPLOAD_STATUS + param.getMd5(), "false");
redisUtil.set(FileConstant.FILE_MD5_KEY + param.getMd5(), uploadDirPath + FileConstant.FILE_SEPARATORCHAR + fileName + ".conf", 10 * 60);
}
return false;
}
/**
* 保存文件操作
*/
public FileUploadDTO saveAndFileUploadDTO(FileUploadRequestDTO param, File tmpFile) {
FileUploadDTO fileUploadDTO = null;
int retryCount = 0;
// 重试机制
while (retryCount < 3) {
try {
fileUploadDTO = removeDestFile(tmpFile, param);
if (fileUploadDTO.getUploadComplete()) {
log.info("file 【{}】 upload complete 【{}】!!", param.getName(), fileUploadDTO.getUploadComplete());
}
return fileUploadDTO;
} catch (Exception e) {
log.error(e.getMessage(), e);
retryCount++;
}
}
return fileUploadDTO;
}
/**
* 移动文件
*
* @param srcFile 将要修改名字的文件
* @param param 请求参数
*/
private FileUploadDTO removeDestFile(File srcFile, FileUploadRequestDTO param) {
//检查要重命名的文件是否存在,是否是文件
FileUploadDTO fileUploadDTO = new FileUploadDTO();
if (!srcFile.exists() || srcFile.isDirectory()) {
log.info("File does not exist: {}", srcFile.getPath());
fileUploadDTO.setUploadComplete(false);
return fileUploadDTO;
}
String destFileName = param.getName();
String ext = FileUtil.extName(destFileName);
String filePath = param.getPath() + FileConstant.FILE_SEPARATORCHAR + destFileName;
File destFile = new File(filePath);
log.info("copy file from 【{}】 to 【{}】", srcFile.getPath(), destFile.getPath());
FileUtil.move(srcFile, destFile, true);
log.info("remove source file 【{}】", srcFile.getParentFile());
FileUtil.del(srcFile.getParentFile());
//修改文件名
fileUploadDTO.setMtime(DateUtil.current());
fileUploadDTO.setUploadComplete(true);
fileUploadDTO.setPath(filePath);
fileUploadDTO.setSize(destFile.length());
fileUploadDTO.setFileExt(ext);
fileUploadDTO.setFileId(destFileName);
return fileUploadDTO;
}
/**
* 构建临时文件存储路径: 指定文件模块路径/asdashjkhkjadas/
* @param param
* @return
*/
public String buildTmpFilePath(FileUploadRequestDTO param) {
String path = param.getPath() + FileUtil.FILE_SEPARATOR + param.getMd5();
return path;
}
}