spring boot - minio 分片上传

spring boot - minio 分片上传

主要分三个接口:

• 分片上传
• 分片合并
• 分片完整性校验

Md5UploadController

@RestController
public class Md5UploadController {

    @Autowired
    private UploadMd5Service uploadMd5Service;

    /**
     * 分片上传
     *
     * @param file       分片file
     * @param fileMd5    完整fileMd5
     * @param chunkIndex 分片id
     * @return
     */
    @PostMapping("/file/upload/chunk")
    public String uploadChunk(@RequestParam("file") MultipartFile file,
                              @RequestParam(value = "fileMd5") String fileMd5,
                              @RequestParam("chunkIndex") Integer chunkIndex) {
        return uploadMd5Service.uploadChunk(file, fileMd5, chunkIndex);
    }

    /**
     * 文件合并
     *
     * @param fileMd5  完整 fileMd5
     * @param fileName 合并后文件名称
     * @param total    合并文件总数【用于校验】
     * @param prefix 分片所在前缀 2023-12/09/fileMd5/chunk/chunk0...
     * @return
     */
    @PostMapping("/file/merge")
    public String merge(@RequestParam(value = "fileMd5") String fileMd5, @RequestParam("fileName") String fileName,
                        @RequestParam("total") int total,
                        @RequestParam(value = "prefix",required = false)String prefix) {
        return uploadMd5Service.merge(fileMd5, fileName, total, prefix);
    }

    /**
     * 分片文件校验
     *
     * @param fileMd5    完整fileMd5
     * @param chunkIndex 分片index
     * @return
     */
    @PostMapping("/file/upload/chunk/check")
    public boolean chunkCheck(@RequestParam(value = "fileMd5") String fileMd5,
                             @RequestParam("chunkIndex") Integer chunkIndex) {
        return uploadMd5Service.chunkCheck(fileMd5, chunkIndex);
    }
}

UploadMd5Service

public interface UploadMd5Service {
    /**
     * 分片上传
     *
     * @param file       分片file
     * @param fileMd5    完整fileMd5
     * @param chunkIndex 分片id
     * @return
     */
    String uploadChunk(MultipartFile file, String fileMd5, Integer chunkIndex);

    /**
     * 文件合并
     *
     * @param fileMd5  完整 fileMd5
     * @param fileName 合并后文件名称
     * @param total    合并文件总数【用于校验】
     * @param prefix
     * @return
     */
    String merge(String fileMd5, String fileName, int total, String prefix);

    /**
     * 分片文件校验
     *
     * @param fileMd5    完整fileMd5
     * @param chunkIndex 分片index
     * @return
     */
    boolean chunkCheck(String fileMd5, Integer chunkIndex);
}


//实现
@Service
public class UploadMd5ServiceImpl implements UploadMd5Service {

    @Autowired
    private MinIoAuthClient minIoAuthClient;

    @Autowired
    private ThreadPoolTaskExecutor executor;

    private static String CHUNK = "/chunk/";
    private static String SUFFIX = "chunk-";

    @Override
    public String uploadChunk(MultipartFile file, String fileMd5, Integer chunkIndex) {
        //2023-06/12/fileMd5/chunk/chunk-0,chunk-1
        String objectName = getPath(fileMd5, null).concat(CHUNK).concat(SUFFIX).concat(chunkIndex.toString());
        return minIoAuthClient.upload(file,objectName,true);
    }

    @Override
    public String merge(String fileMd5, String fileName, int total, String prefix) {
        //查询该 路径下所有的分片
        List<String> fuzzyObject = minIoAuthClient.fuzzyListObjects(SUFFIX,getPath(fileMd5,prefix).concat(CHUNK));
        if (CollectionUtils.isEmpty(fuzzyObject) || total != fuzzyObject.size()) {
            throw new RuntimeException("合并失败，分片数不一致!!");
        }
        String pathPrefix = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM" + "/dd"));
        String result = minIoAuthClient.composeObject(fuzzyObject, fileName, pathPrefix);
        if (StrUtil.isNotBlank(result)){
            //如果merge成功,删除分片
            minIoAuthClient.deleteObject(fuzzyObject);
            return pathPrefix.concat(fileMd5);

        }
        return null;
    }

    @Override
    public boolean chunkCheck(String fileMd5, Integer chunkIndex) {
        //2023-06/12/fileMd5/chunk/chunk-0,chunk-1
       return minIoAuthClient.checkObjectExist(getPath(fileMd5, null).concat(CHUNK).concat(SUFFIX).concat(chunkIndex.toString()));
    }

    private String getPath(String fileMd5, String prefix){
        if (StrUtil.isNotBlank(prefix)){
            if (prefix.endsWith("/")){
                return prefix.concat(fileMd5);
            }
            return prefix.concat("/").concat(fileMd5);
        }
        return LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM" + "/dd")).concat("/").concat(fileMd5);
    }
}

MinioAuthClient

@Slf4j
@Component
public class MinIoAuthClient {

    @Autowired
    private MinioConfig config;

    @Autowired
    private MinioClient minioClient;

    @Autowired
    private MinioAsyncClient minioAsyncClient;

    /**
     * 检验文件是否存在
     * @param objectName
     * @return
     */
    public boolean checkObjectExist(String objectName){
        try (InputStream stream = minioClient.getObject(GetObjectArgs.builder()
                .bucket(config.getBucketName()).object(objectName).build())) {
            if (stream!=null){
                return true;
            }
        } catch (Exception e) {
            log.error("####### checkObjectExist error: {}",objectName, e);
        }
        return false;
    }


    /**
     * 单个文件上传
     * @param file
     * @param rename
     * @return
     */
    public String upload(MultipartFile file,String inputFileName, Boolean rename) {
        String objectName = null;
        InputStream inputStream = null;
        try {
            String originalFilename = inputFileName!=null? inputFileName: file.getOriginalFilename();
            if (!rename) {
                String fileName = UUID.randomUUID().toString().replace("-", "")
                                + originalFilename.substring(originalFilename.lastIndexOf("."));
                objectName = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM/dd")).concat("/").concat(fileName);
            } else {
                objectName = originalFilename;
            }
            inputStream = file.getInputStream();
            PutObjectArgs objectArgs = PutObjectArgs.builder().bucket(config.getBucketName()).object(objectName).stream(inputStream, file.getSize(), -1).contentType(file.getContentType()).build();
            minioClient.putObject(objectArgs);

        } catch (Exception e) {
            e.printStackTrace();
            return null;
        } finally {
            if (inputStream != null) {
                try {
                    inputStream.close();
                } catch (IOException e) {
                    log.error("delete file error", e);
                }
            }
        }
        return objectName;
    }

    /**
     * 同步合并文件
     * @param list
     * @param originalFilename 合并后文件名称
     * @param pathPrefix 文件所在路径
     * @return
     */
    public String composeObject(List<String> list, String originalFilename,String pathPrefix) {
        String objectName = null;
        List<ComposeSource> composeSourceList = list.stream().map(s -> {
            ComposeSource source = ComposeSource.builder().bucket(config.getBucketName()).object(s).build();
            return source;
        }).collect(Collectors.toList());
        try {
            //需要按名称排序合并 xx.part0,xx.part1...
            Collections.sort(composeSourceList, new Comparator<>() {
                @Override
                public int compare(ComposeSource o1, ComposeSource o2) {
                    if (o1.object().compareTo(o2.object()) < 0) {
                        return -1;
                    }
                    return 1;
                }
            });
            String fileName = originalFilename;
            objectName = pathPrefix.concat("/").concat(fileName);
            minioClient.composeObject(ComposeObjectArgs.builder().bucket(config.getBucketName()).object(objectName).sources(composeSourceList).build());
        } catch (Exception e) {
            log.error("####### composeObject error", e);
            return null;
        }
        return objectName;
    }

    public void deleteObject(List<String> objects){
        List<DeleteObject> collect = objects.stream().map(str -> new DeleteObject(str)).collect(Collectors.toList());
        Iterable<Result<DeleteError>> results = minioClient.removeObjects(RemoveObjectsArgs.builder().bucket(config.getBucketName()).objects(collect).build());
        Iterator<Result<DeleteError>> iterator = results.iterator();
        while (iterator.hasNext()){
            Result<DeleteError> next = iterator.next();
            System.out.println(next);
        }
    }

    /**
     * minio已经存在
     * 获取一个指定了 HTTP 方法、到期时间和自定义请求参数的对象URL地址
     *
     * @param objectName
     * @param timeOut
     * @param timeUnit
     * @return
     */
    public String getPresignedObjectUrl(String objectName, int timeOut, TimeUnit timeUnit) {
        try {
            GetPresignedObjectUrlArgs.Builder builder = GetPresignedObjectUrlArgs.builder().method(Method.GET).bucket(config.getBucketName()).object(objectName);
            if (timeOut > 0) {
                builder.expiry(timeOut, timeUnit);
            }
            return minioClient.getPresignedObjectUrl(builder.build());
        } catch (Exception e) {
            log.error("###### getPresignedObjectUrl error: {}", objectName, e);
            return null;
        }
    }

    /**
     * 获取文件预生成地址，【minio上文件不存在】
     * 一般给合并文件使用
     * @param objectName
     * @param queryParams
     * @param timeOut
     * @param timeUnit
     * @return
     */
    public String getPreviewObjectUrl(String objectName,Map<String,String>queryParams, int timeOut, TimeUnit timeUnit) {
        try {
            GetPresignedObjectUrlArgs.Builder builder =
                    GetPresignedObjectUrlArgs.builder().method(Method.PUT).bucket(config.getBucketName()).object(objectName).extraQueryParams(queryParams);
            if (timeOut > 0) {
                builder.expiry(timeOut, timeUnit);
            }
            return minioClient.getPresignedObjectUrl(builder.build());
        } catch (Exception e) {
            log.error("###### getPreviewObjectUrl error: {}", objectName, e);
            return null;
        }
    }

    /**
     * 递归根据文件名称前缀模糊查询
     * @param objectName
     * @return
     */
    public List<String> fuzzyListObjects(String objectName) {
        try {
            Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(config.getBucketName()).recursive(true).build());
            if (results == null || !results.iterator().hasNext()) {
                return null;
            }
            List<String> list = new ArrayList<>();
            for (Result<Item> result : results) {
                String s = result.get().objectName();
                String finalName = null;
                if (s.lastIndexOf("/") != -1) {
                    finalName = s.substring(s.lastIndexOf("/") + 1);
                } else {
                    finalName = s;
                }
                if (!finalName.startsWith(objectName)) {
                    continue;
                }
                list.add(s);
            }
            return list;
        } catch (Exception e) {
            log.error("#### fuzzyListObjects error: {}", objectName, e);
            return null;
        }
    }

    /**
     * 模糊查询某个目录下 objectName开头的文件, 一般查询所有分片
     * @param objectName 文件名称
     * @param pathPrefix 路径前缀
     * @return
     */
    public List<String> fuzzyListObjects(String objectName,String pathPrefix) {
        try {
            Iterable<Result<Item>> results =
                    minioClient.listObjects(ListObjectsArgs.builder().bucket(config.getBucketName()).prefix(pathPrefix).build());
            if (results == null || !results.iterator().hasNext()) {
                return null;
            }
            List<String> list = new ArrayList<>();
            for (Result<Item> result : results) {
                String s = result.get().objectName();
                String finalName = null;
                if (s.lastIndexOf("/") != -1) {
                    finalName = s.substring(s.lastIndexOf("/") + 1);
                } else {
                    finalName = s;
                }
                if (!finalName.startsWith(objectName)) {
                    continue;
                }
                list.add(s);
            }
            return list;
        } catch (Exception e) {
            log.error("#### fuzzyListObjects error: {}", objectName, e);
            return null;
        }
    }

}

测试

上传分片

merge:

合并成功！

good luck!!