分片上传文件到minio

一、导入pom

<dependency>
    <groupId>io.minio</groupId>
    <artifactId>minio</artifactId>
    <version>8.5.11</version>
</dependency>

二、添加application.yml配置

minio:
  endpoint: http://127.0.0.1:8080 #minio地址
  accessKey: xxx    #账号
  secretKey: xxx    #密码
  bucketName: xxxx  #桶名

 三、添加MinioConfiguration文件

package com.siteshun.hospitalarchives.config;

import io.minio.MinioClient;
import lombok.Data;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Data
@Configuration
public class MinioConfiguration {

    /**
     * 访问地址
     */
    @Value("${minio.endpoint}")
    private String endpoint;

    /**
     * accessKey类似于用户ID，用于唯一标识你的账户
     */
    @Value("${minio.accessKey}")
    private String accessKey;

    /**
     * secretKey是你账户的密码
     */
    @Value("${minio.secretKey}")
    private String secretKey;

    /**
     * 默认存储桶
     */
    @Value("${minio.bucketName}")
    private String bucketName;

    @Bean
    public MinioClient minioClient() {
        MinioClient minioClient = MinioClient.builder()
                .endpoint(endpoint)
                .credentials(accessKey, secretKey)
                .build();
        return minioClient;
    }
}

四、上传方法

package com.siteshun.hospitalarchives.utils;

import io.minio.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;


/**
 * MinIO工具类
 */
@Slf4j
@Component
@RequiredArgsConstructor
public class MinioUtils {

    @Autowired
    private MinioClient minioClient;
    
    /**
     * 分片上传文件到minio
     *
     * @param bucketName  存储桶
     * @param objectName  上传到minio的文件名
     * @param inputStream 文件流
     * @param fileSize 文件大小
     * @return
     */
    @SneakyThrows(Exception.class)
    public ObjectWriteResponse shardingUploadFile(String bucketName, String objectName, InputStream inputStream,long fileSize) {
        // 分片上传
        List<ComposeSource> sources = splitFile(inputStream, bucketName, objectName, 5 * 1024 * 1024,fileSize);
        //合并
        log.info("开始合并------------");
        ObjectWriteResponse response = minioClient.composeObject(ComposeObjectArgs.builder()
                .bucket(bucketName)
                .object(objectName)
                .sources(sources)
                .build());
        log.info("合并完成------------");
        //删除分片
        for (ComposeSource source : sources) {
            minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(source.object()).build());
        }
        return response;
    }

    /**
     * 将给定的大文件分割成多个小文件，每个小文件的大小不超过指定的分片大小，
     * 并直接将这些分片上传到MinIO。
     *
     * @param in             文件输入流
     * @param bucketName     存储桶名称
     * @param objectName     上传到minio名称
     * @param chunkSizeBytes 每个分片的大小（以字节为单位）
     * @param fileSize       文件大小
     */
    public List<ComposeSource> splitFile(InputStream in, String bucketName, String objectName, long chunkSizeBytes, long fileSize) {
        // 用于存储每个部分的响应
        List<ComposeSource> sourceList = new ArrayList<>();
        try {
            int bytesRead;
            int fileCounter = 1;
            byte[] buffer = new byte[(int) chunkSizeBytes];
            ByteArrayOutputStream tempBuffer = new ByteArrayOutputStream();
            while ((bytesRead = in.read(buffer)) != -1) {
                // 上传当前分片
                int finalFileCounter = fileCounter;
                int finalBytesRead = bytesRead;
                // 当前分片数+1乘分片大小，如果小于文件大小，则说明不是倒数第二个分片直接存minio，如果大于文件大小，则说明是倒数第二、一个分片，将剩余的数据存入tempBuffer
                if ((finalFileCounter + 1) * chunkSizeBytes < fileSize) {
                    try {
                        uploadChunkToMinio(sourceList, bucketName, objectName + "_part_" + finalFileCounter + ".dat", buffer, (int) finalBytesRead);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                } else {
                    tempBuffer.write(buffer, 0, finalBytesRead);
                }
                fileCounter++;
            }
            // 上传剩余的数据
            if (tempBuffer.size() > 0) {
                try {
                    uploadChunkToMinio(sourceList, bucketName, objectName + "_part_" + fileSize / chunkSizeBytes + ".dat", tempBuffer.toByteArray(), tempBuffer.size());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sourceList;
    }


    /**
     * 上传分片到MinIO
     *
     * @param sourceList
     * @param bucketName
     * @param objectName
     * @param buffer
     * @param length
     * @return
     * @author 封朝韬
     * @date 2024-08-07
    */
    private void uploadChunkToMinio(List<ComposeSource> sourceList, String bucketName, String objectName, byte[] buffer, int length) throws Exception {
        log.info("上传分片：{},大小：{}" ,objectName,length);
        // 创建一个基于buffer的输入流
        InputStream inputStream = new ByteArrayInputStream(buffer, 0, length);
        //上传分片
        minioClient.putObject(PutObjectArgs.builder()
                .bucket(bucketName)
                .object(objectName)
                .stream(inputStream, length, -1)
                .build());
        // 将分片信息添加到sourceList中
        sourceList.add(ComposeSource.builder().bucket(bucketName).object(objectName).build());
    }
}