Minio大文件分片上传

最新推荐文章于 2025-08-10 18:26:29 发布

原创最新推荐文章于 2025-08-10 18:26:29 发布 · 285 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java #minio

一、引入依赖

		<dependency>
			<groupId>io.minio</groupId>
			<artifactId>minio</artifactId>
			<version>8.3.3</version>
		</dependency>

二、自定义Minio客户端

package com.gstanzer.video.controller;

import com.google.common.collect.Multimap;
import io.minio.*;
import io.minio.errors.*;
import io.minio.messages.Part;

import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;

public class CustomMinioClient extends MinioClient {

    /**
     * 继承父类
     */
    public CustomMinioClient(MinioClient client) {
        super(client);
    }

    /**
     * 初始化分片上传即获取uploadId
     */
    public String initMultiPartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
        CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams);
        return response.result().uploadId();
    }

    /**
     * 上传单个分片
     */
    public UploadPartResponse uploadMultiPart(String bucket, String region, String object, Object data,
                                              long length,
                                              String uploadId,
                                              int partNumber,
                                              Multimap<String, String> headers,
                                              Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException {
        return this.uploadPart(bucket, region, object, data, length, uploadId, partNumber, headers, extraQueryParams);
    }

    /**
     * 合并分片
     */
    public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws IOException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException, ServerException, InvalidKeyException {
        return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams);
    }

    public void cancelMultipartUpload(String bucketName, String region, String objectName, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, IOException, InvalidKeyException, XmlParserException, InvalidResponseException, InternalException {
        this.abortMultipartUpload(bucketName, region, objectName, uploadId, extraHeaders, extraQueryParams);
    }

    /**
     * 查询当前上传后的分片信息
     */
    public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException {
        return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams);
    }
}

三、分片上传核心完整代码

1.实体类

package com.gstanzer.video.form;

import lombok.Data;

/**
 * @author: tangbingbing
 * @date: 2025/6/4 08:27
 */
@Data
public class UploadPartForm {
    // 分片文件路径拼接字符串（比如：E:\\scdx\\test\\test.doc.part1,E:\\scdx\\test\\test.doc.part2）
    private String partFilePaths;
    // 分片文件url
    private String partFileUrl;
    // 上传id
    private String uploadId;
    // 文件类型（word类型：application/msword，安卓安装包类型：application/vnd.android.package-archive）
    private String contentType;
}

2.控制层

package com.gstanzer.video.controller;

import com.alibaba.fastjson.JSON;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.gstanzer.video.form.UploadPartForm;
import com.gstanzer.video.swagger.ApiForBackEndInVideo;
import io.minio.*;
import io.minio.http.Method;
import io.minio.messages.Part;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.FileEntity;
import org.apache.http.impl.client.HttpClients;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;

import javax.annotation.PostConstruct;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

@Slf4j
@RestController
@RequestMapping("/minio")
public class MinioController {

    private CustomMinioClient minioClient;

    @Value("${minio.endpoint}")
    private String endpoint;
    @Value("${minio.bucketName}")
    private String bucketName;
    @Value("${minio.access-key}")
    private String accessKey;
    @Value("${minio.secret-key}")
    private String secretKey;

    @PostConstruct
    public void init() throws Exception {
        minioClient = new CustomMinioClient(MinioClient.builder()
                .endpoint(endpoint)
                .credentials(accessKey, secretKey)
                .build());

        boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
        if (!found) {
            log.info("Not found minio bucket: {}, auto create it now", bucketName);
            minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
            found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
            log.info("Auto create minio bucket: {} {}", bucketName, found);
        } else {
            log.info("Found minio bucket: {}", bucketName);
        }
    }

    /**
     * 第一步，简单用一个10MB出头的文件，按5MB分片大小进行分片
     */
    public static void main(String[] args) throws Exception {
        long CHUNK_SIZE = 15 * 1024 * 1024;
        // 将文件分片存储
        String filePath = "E:\\scdx\\app\\xfgj_v3.0.88.apk";
        File file = new File(filePath);
        long fileSize = file.length();
        int chunkCount = (int) Math.ceil((double) fileSize / CHUNK_SIZE);
        log.info("Created: " + chunkCount + " chunks");
        try (FileInputStream fis = new FileInputStream(file)) {
            byte[] buffer = new byte[(int) CHUNK_SIZE];
            for (int i = 0; i < chunkCount; i++) {
                String chunkFileName = filePath + ".part" + (i + 1);
                try (FileOutputStream fos = new FileOutputStream(chunkFileName)) {
                    int bytesRead = fis.read(buffer);
                    fos.write(buffer, 0, bytesRead);
                    log.info("Created: " + chunkFileName + " (" + bytesRead + " bytes)");
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 第二步，申请一个大文件上传
     */
    @ApiForBackEndInVideo
    @PostMapping("/uploadLargeFile/apply")
    public ResponseEntity<Map> applyUploadPsiResult2Minio(@RequestParam("largeFileName") String largeFileName,
                                                          @RequestParam("chunkCount") Integer chunkCount) {
        String uploadId = getUploadId(largeFileName, null);
        Map<String, Object> map = new HashMap<>();
        map.put("uploadId", uploadId);
        Map<String, String> reqParams = new HashMap<>();
        reqParams.put("uploadId", uploadId);
        List<String> uploadUrlList = new ArrayList<>();
        for (int i = 1; i <= chunkCount; i++) {
            reqParams.put("partNumber", String.valueOf(i));
            String uploadUrl = getPresignedObjectUrl(largeFileName, reqParams);
            uploadUrlList.add(uploadUrl);
        }
        map.put("chunkUploadUrls", uploadUrlList);
        return ResponseEntity.ok(map);
    }

    /**
     * 准备分片上传时，在此先获取上传任务id
     */
    private String getUploadId(String objectName, String contentType){
        log.info("tip message: 通过 <{}-{}-{}> 开始初始化<分片上传>数据", objectName, contentType, bucketName);
        if (StringUtils.isBlank(contentType)) {
            contentType = "application/octet-stream";
        }
        HashMultimap<String, String> headers = HashMultimap.create();
        headers.put("Content-Type", contentType);
        try {
            return minioClient.initMultiPartUpload(bucketName, null, objectName, headers, null);
        } catch (Exception e) {
            throw new RuntimeException("获取uploadId失败", e);
        }
    }

    /**
     * 请求分片上传的url
     */
    private String getPresignedObjectUrl(String fileName, Map<String, String> reqParams) {
        try {
            String presignedObjectUrl = minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder()
                    .method(Method.PUT)
                    .bucket(bucketName)
                    .object(fileName)
                    .expiry(1, TimeUnit.DAYS)
                    .extraQueryParams(reqParams)
                    .build());
            return presignedObjectUrl;
        } catch (Exception e) {
            throw new RuntimeException("getPresignedObjectUrl failed", e);
        }
    }

    /**
     * 第三步，每个分片逐个通过签名后的分片上传url，进行上传，未合并之前都可以重复覆盖上传
     * 在video服务的控制台通过如下命令将分片下载到video服务中
     * wget http://100.86.2.1:80/xfaq/xfgj_v3.0.88.apk.part1
     */
    @ApiForBackEndInVideo
    @PostMapping("/uploadLargeFile/uploadPart")
    public ResponseEntity<String> applyUploadPsiResult2Minio(@RequestBody UploadPartForm form) {
        String partFilePaths = form.getPartFilePaths();
        String partFileUrl = form.getPartFileUrl();
        try {
            // 第一步得到的分片文件
            List<String> chunkFilePaths = Lists.newArrayList(partFilePaths.split(","));
            // 第二步得到的上传url信息
            Map<String, Object> uploadIdMap = new HashMap<>();
            uploadIdMap.put("uploadId", form.getUploadId());
            List<String> chunkUploadUrls = Lists.newArrayList(partFileUrl.split(","));
            uploadIdMap.put("chunkUploadIds", chunkUploadUrls);
            // 基于此，分片直接上传到minio，不走服务端，省去一次网络IO开销
            HttpClient httpClient = HttpClients.createDefault();
            for (int i = 0; i < chunkUploadUrls.size(); i++) {
                // PUT直接上传到minio
                String chunkUploadId = chunkUploadUrls.get(i);
                HttpPut httpPut = new HttpPut(chunkUploadId);
                httpPut.setHeader("Content-Type",form.getContentType());
                File chunkFile = new File(chunkFilePaths.get(i));
                FileEntity chunkFileEntity = new FileEntity(chunkFile);
                httpPut.setEntity(chunkFileEntity);
                HttpResponse chunkUploadResp = httpClient.execute(httpPut);
                log.info("[分片" + (i + 1) + "]上传响应：" + JSON.toJSONString(chunkUploadResp));
                httpPut.releaseConnection();
            }
            return ResponseEntity.ok("上传成功！");
        } catch (IOException e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        }
    }

    public static void main3(String[] args) throws Exception {
        // 第一步得到的分片文件
        List<String> chunkFilePaths = Lists.newArrayList(
                "E:\\scdx\\test\\test.doc.part1",
                "E:\\scdx\\test\\test.doc.part2"
        );
        // 第二步得到的上传url信息
        Map<String, Object> uploadIdMap = new HashMap<>();
        uploadIdMap.put("uploadId", "YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw");
        List<String> chunkUploadUrls = Lists.newArrayList(
                "http://127.0.0.1:9000/xfaq/test.doc?uploadId=YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20250603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250603T075657Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=649af2001e096f422479e139efc4aab51301061c026f40c0685cd20edbb28211",
                "http://127.0.0.1:9000/xfaq/test.doc?uploadId=YTJlOWZhMmEtM2I3My00MmIyLWE0YjgtMDFkYjQzMzIyNmVhLjFmYzQ2M2ViLTI2YmYtNDZjMi04M2ZlLWJjMThjOTk0MWU4OHgxNzQ4OTM3NDE3MjI4ODA1MTAw&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20250603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250603T075657Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=eedd5431a6e4ba0ebda1c2b227f7543eed77d245b3d86c6d5d505d27f61cd823");
        uploadIdMap.put("chunkUploadIds", chunkUploadUrls);
        // 基于此，分片直接上传到minio，不走服务端，省去一次网络IO开销
        HttpClient httpClient = HttpClients.createDefault();
        for (int i = 0; i < chunkUploadUrls.size(); i++) {
            // PUT直接上传到minio
            String chunkUploadId = chunkUploadUrls.get(i);
            HttpPut httpPut = new HttpPut(chunkUploadId);
            httpPut.setHeader("Content-Type","application/msword");
            File chunkFile = new File(chunkFilePaths.get(i));
            FileEntity chunkFileEntity = new FileEntity(chunkFile);
            httpPut.setEntity(chunkFileEntity);
            HttpResponse chunkUploadResp = httpClient.execute(httpPut);
            log.info("[分片" + (i + 1) + "]上传响应：" + JSON.toJSONString(chunkUploadResp));
            httpPut.releaseConnection();
        }
    }

    /**
     * 第四步，合并分片
     */
    @ApiForBackEndInVideo
    @PostMapping("/uploadLargeFile/merge")
    public ResponseEntity<Boolean> applyUploadPsiResult2Minio(@RequestParam("largeFileName") String largeFileName,
                                                              @RequestParam("uploadId") String uploadId) {
        boolean mergeResult = mergeMultipartUpload(largeFileName, uploadId);
        return ResponseEntity.ok(mergeResult);
    }

    /**
     * 分片上传完后合并
     * @param objectName 文件名称
     * @param uploadId
     * @return
     */
    public boolean mergeMultipartUpload(String objectName, String uploadId) {
        try {
            log.info("ready to merge <" + objectName + " - " + uploadId + " - " + bucketName + ">");
            // 查询上传后的分片数据
            ListPartsResponse partResult = minioClient.listMultipart(bucketName, null, objectName, 1000, 0, uploadId, null, null);
            int chunkCount = partResult.result().partList().size();
            Part[] parts = new Part[chunkCount];
            int partNumber = 1;
            for (Part part : partResult.result().partList()) {
                parts[partNumber - 1] = new Part(partNumber, part.etag());
                partNumber++;
            }
            // 合并分片
            ObjectWriteResponse objectWriteResponse = minioClient.mergeMultipartUpload(bucketName, null, objectName, uploadId, parts, null, null);
            log.info("mergeMultipartUpload resp: {}", objectWriteResponse);
        } catch (Exception e) {
            log.error("合并失败：", e);
            return false;
        }
        return true;
    }
}