基于minio的多线程分片文件上传-进度条显示

读取须知：需掌握minio、多线程、SSE的使用

一、minio配置

1、minio配置

1.1、配置

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.worldintek.fms.template.MinioTemplate;
import io.minio.MinioClient;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;


@Configuration
@Data
@ConfigurationProperties(value = "minio")
public class MinioConfig {


    /**
     * 对象存储服务的对外URL
     */
    private String externalAccess;

    /**
     * 对象存储服务的URL
     */
    private String endpoint;

    /**
     * Access key就像用户ID，可以唯一标识你的账户。
     */
    private String accessKey;

    /**
     * Secret key是你账户的密码。
     */
    private String secretKey;

    /**
     * bucketName是你设置的桶的名称
     */
    private String bucketName;

    /**
     * 初始化一个MinIO客户端用来连接MinIO存储服务
     *
     * @return MinioClient
     */
    @Bean(name = "minioClient")
    public MinioClient initMinioClient() {
        return MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build();
    }

    /**
     * 初始化MinioTemplate，封装了一些MinIOClient的基本操作
     *
     * @return MinioTemplate
     */
    @Bean(name = "minioTemplate")
    public MinioTemplate minioTemplate() {
        return new MinioTemplate(initMinioClient(), this);
    }

    @JsonIgnore
    public String getPublicUrlPrefix(){
        return externalAccess + "/" + bucketName;
    }
}

1.2、模板

package com.worldintek.fms.template;

import com.worldintek.fms.config.MinioConfig;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import com.worldintek.fms.domain.MinioTemplateResult;
import io.minio.*;
import io.minio.http.Method;
import io.minio.messages.Bucket;
import io.minio.messages.Item;
import lombok.AllArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;

import javax.annotation.PostConstruct;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * @author wxl
 * @description:
 * @version: v1.0
 * @since 2022-05-03 13:04
 */
@Slf4j
@AllArgsConstructor
public class MinioTemplate {
    /**
     * MinIO 客户端
     */
    private final MinioClient minioClient;

    /**
     * MinIO 配置类
     */
    private final MinioConfig minioConfig;

    /**
     * 查询所有存储桶
     *
     * @return Bucket 集合
     */
    @SneakyThrows
    public List<Bucket> listBuckets() {
        return minioClient.listBuckets();
    }

    /**
     * 桶是否存在
     *
     * @param bucketName 桶名
     * @return 是否存在
     */
    @SneakyThrows
    public boolean bucketExists(String bucketName) {
        return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
    }

    /**
     * 创建存储桶
     *
     * @param bucketName 桶名
     */
    @SneakyThrows
    public void makeBucket(String bucketName) {
        if (!bucketExists(bucketName)) {
            minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
        }
    }

    /**
     * 删除一个空桶 如果存储桶存在对象不为空时，删除会报错。
     *
     * @param bucketName 桶名
     */
    @SneakyThrows
    public void removeBucket(String bucketName) {
        removeBucket(bucketName, false);
        minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
    }

    /**
     * 删除一个桶 根据桶是否存在数据进行不同的删除
     * 桶为空时直接删除
     * 桶不为空时先删除桶中的数据，然后再删除桶
     *
     * @param bucketName 桶名
     */
    @SneakyThrows
    public void removeBucket(String bucketName, boolean bucketNotNull) {
        if (bucketNotNull) {
            deleteBucketAllObject(bucketName);
        }
        minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
    }

    /**
     * @description: 创建文件夹
     * @author wxl
     * @date 2022/7/21 17:13
     * @param
     * @return
     **/
    @SneakyThrows
    public MinioTemplateResult createDir(String dirName, String bucketName){
        minioClient.putObject(
                PutObjectArgs.builder()
                        .bucket(bucketName)
                        .stream(new ByteArrayInputStream(new byte[] {}), 0, -1)
                        .object(dirName + "/")
                        .build());
        return new MinioTemplateResult(dirName, null);
    }




    /**
     * 上传文件
     *
     * @param inputStream      流
     * @param originalFileName 原始文件名
     * @param bucketName       桶名
     * @return ObjectWriteResponse
     */
    @SneakyThrows
    public MinioTemplateResult putSimpleObject(InputStream inputStream, String bucketName, String originalFileName) {
        try {
            if (ObjectUtils.isEmpty(bucketName)) {
                bucketName = minioConfig.getBucketName();
            }
            minioClient.putObject(
                    PutObjectArgs.builder()
                            .bucket(bucketName)
                            .object(originalFileName)
                            .stream(inputStream, inputStream.available(), -1)
                            .build());


            return new MinioTemplateResult(originalFileName, originalFileName);
        } finally {
            if (inputStream != null) {
                inputStream.close();
            }
        }
    }

    /**
     * 上传文件
     *
     * @param inputStream      流
     * @param originalFileName 原始文件名
     * @param bucketName       桶名
     * @return ObjectWriteResponse
     */
    @SneakyThrows
    public MinioTemplateResult putObject(InputStream inputStream, String bucketName, String originalFileName) {
        String uuidFileName = generateFileInMinioName(originalFileName);
        try {
            if (ObjectUtils.isEmpty(bucketName)) {
                bucketName = minioConfig.getBucketName();
            }
            minioClient.putObject(
                    PutObjectArgs.builder()
                            .bucket(bucketName)
                            .object(uuidFileName)
                            .stream(inputStream, inputStream.available(), -1)
                            .build());


            return new MinioTemplateResult(uuidFileName, originalFileName);
        } finally {
            if (inputStream != null) {
                inputStream.close();
            }
        }
    }


    /**
     * 删除桶中所有的对象
     *
     * @param bucketName 桶对象
     */
    @SneakyThrows
    public void deleteBucketAllObject(String bucketName) {
        List<String> list = listObjectNames(bucketName);
        if (!list.isEmpty()) {
            for (String objectName : list) {
                deleteObject(bucketName, objectName);
            }
        }
    }

    /**
     * 查询桶中所有的对象名
     *
     * @param bucketName 桶名
     * @return objectNames
     */
    @SneakyThrows
    public List<String> listObjectNames(String bucketName) {
        List<String> objectNameList = new ArrayList<>();
        if (bucketExists(bucketName)) {
            Iterable<Result<Item>> results = listObjects(bucketName, true);
            for (Result<Item> result : results) {
                String objectName = result.get().objectName();
                objectNameList.add(objectName);
            }
        }
        return objectNameList;
    }


    /**
     * 删除一个对象
     *
     * @param bucketName 桶名
     * @param objectName 对象名
     */
    @SneakyThrows
    public void deleteObject(String bucketName, String objectName) {
        minioClient.removeObject(RemoveObjectArgs.builder()
                .bucket(bucketName)
                .object(objectName)
                .build());
    }

    /**
     * 上传分片文件
     *
     * @param inputStream 流
     * @param objectName  存入桶中的对象名
     * @param bucketName  桶名
     * @return ObjectWriteResponse
     */
    @SneakyThrows
    public MinioTemplateResult putChunkObject(InputStream inputStream, String bucketName, String objectName) {
        try {
            minioClient.putObject(
                    PutObjectArgs.builder()
                            .bucket(bucketName)
                            .object(objectName)
                            .stream(inputStream, inputStream.available(), -1)
                            .build());
            return new MinioTemplateResult(objectName, objectName);
        } finally {
            if (inputStream != null) {
                inputStream.close();
            }
        }
    }

    /**
     * 返回临时带签名、Get请求方式的访问URL
     *
     * @param bucketName 桶名
     * @param filePath   Oss文件路径
     * @return 临时带签名、Get请求方式的访问URL
     */
    @SneakyThrows
    public String getPresignedObjectUrl(String bucketName, String filePath) {
        return minioClient.getPresignedObjectUrl(
                GetPresignedObjectUrlArgs.builder()
                        .method(Method.GET)
                        .bucket(bucketName)
                        .object(filePath)
                        .build());
    }

    /**
     * 返回临时带签名、过期时间为1天的PUT请求方式的访问URL
     *
     * @param bucketName  桶名
     * @param filePath    Oss文件路径
     * @param queryParams 查询参数
     * @return 临时带签名、过期时间为1天的PUT请求方式的访问URL
     */
    @SneakyThrows
    public String getPresignedObjectUrl(String bucketName, String filePath, Map<String, String> queryParams) {
        return minioClient.getPresignedObjectUrl(
                GetPresignedObjectUrlArgs.builder()
                        .method(Method.PUT)
                        .bucket(bucketName)
                        .object(filePath)
                        .expiry(1, TimeUnit.DAYS)
                        .extraQueryParams(queryParams)
                        .build());
    }


    /**
     * GetObject接口用于获取某个文件（Object）。此操作需要对此Object具有读权限。
     *
     * @param bucketName 桶名
     * @param objectName 文件路径
     */
    @SneakyThrows
    public InputStream getObject(String bucketName, String objectName) {
        return minioClient.getObject(
                GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
    }

    /**
     * 查询桶的对象信息
     *
     * @param bucketName 桶名
     * @param recursive  是否递归查询
     * @return 桶的对象信息
     */
    @SneakyThrows
    public Iterable<Result<Item>> listObjects(String bucketName, boolean recursive) {
        return minioClient.listObjects(
                ListObjectsArgs.builder().bucket(bucketName).recursive(recursive).build());
    }

    /**
     * 获取带签名的临时上传元数据对象，前端可获取后，直接上传到Minio
     *
     * @param bucketName 桶名称
     * @param fileName   文件名
     * @return Map<String, String>
     */
    @SneakyThrows
    public Map<String, String> getPresignedPostFormData(String bucketName, String fileName) {
        // 为存储桶创建一个上传策略，过期时间为7天
        PostPolicy policy = new PostPolicy(bucketName, ZonedDateTime.now().plusDays(1));
        // 设置一个参数key，值为上传对象的名称
        policy.addEqualsCondition("key", fileName);
        // 添加Content-Type，例如以"image/"开头，表示只能上传照片，这里吃吃所有
        policy.addStartsWithCondition("Content-Type", MediaType.ALL_VALUE);
        // 设置上传文件的大小 64kiB to 10MiB.
        //policy.addContentLengthRangeCondition(64 * 1024, 10 * 1024 * 1024);
        return minioClient.getPresignedPostFormData(policy);
    }


    public String generateFileInMinioName(String originalFilename) {
        return "files" + StrUtil.SLASH + DateUtil.format(new Date(), "yyyy-MM-dd") + StrUtil.SLASH + UUID.randomUUID() + StrUtil.UNDERLINE + originalFilename;
    }

    /**
     * 初始化默认存储桶
     */
    @PostConstruct
    public void initDefaultBucket() {
        String defaultBucketName = minioConfig.getBucketName();
        if (bucketExists(defaultBucketName)) {
            log.info("默认存储桶：{}已存在", defaultBucketName);
        } else {
            log.info("创建默认存储桶：{}", defaultBucketName);
            makeBucket(minioConfig.getBucketName());
        }
        ;
    }


    /**
     * 文件合并，将分块文件组成一个新的文件
     *
     * @param bucketName       合并文件生成文件所在的桶
     * @param fileName         原始文件名
     * @param sourceObjectList 分块文件集合
     * @return MinioTemplateResult
     */
    @SneakyThrows
    public MinioTemplateResult composeObject(String bucketName, String fileName, List<ComposeSource> sourceObjectList) {
        String filenameExtension = StringUtils.getFilenameExtension(fileName);
        String objectName = UUID.randomUUID() + "." + filenameExtension;
        minioClient.composeObject(ComposeObjectArgs.builder()
                .bucket(bucketName)
                .object(objectName)
                .sources(sourceObjectList)
                .build());

        String presignedObjectUrl = getPresignedObjectUrl(bucketName, fileName);
        return new MinioTemplateResult(presignedObjectUrl, fileName);
    }


    /**
     * 文件合并，将分块文件组成一个新的文件
     *
     * @param bucketName       合并文件生成文件所在的桶
     * @param objectName       原始文件名
     * @param sourceObjectList 分块文件集合
     * @return MinioTemplateResult
     */
    @SneakyThrows
    public MinioTemplateResult composeObject(List<ComposeSource> sourceObjectList, String bucketName, String objectName) {
        minioClient.composeObject(ComposeObjectArgs.builder()
                .bucket(bucketName)
                .object(objectName)
                .sources(sourceObjectList)
                .build());
        String presignedObjectUrl = getPresignedObjectUrl(bucketName, objectName);
        return new MinioTemplateResult(presignedObjectUrl, objectName);
    }

    /**
     * 文件合并，将分块文件组成一个新的文件
     *
     * @param originBucketName 分块文件所在的桶
     * @param targetBucketName 合并文件生成文件所在的桶
     * @param objectName       存储于桶中的对象名
     * @return MinioTemplateResult
     */
    @SneakyThrows
    public MinioTemplateResult composeObject(String originBucketName, String targetBucketName, String objectName) {

        Iterable<Result<Item>> results = listObjects(originBucketName, true);
        List<String> objectNameList = new ArrayList<>();
        for (Result<Item> result : results) {
            Item item = result.get();
            objectNameList.add(item.objectName());
        }


        if (ObjectUtils.isEmpty(objectNameList)) {
            throw new IllegalArgumentException(originBucketName + "桶中没有文件，请检查");
        }

        List<ComposeSource> composeSourceList = new ArrayList<>(objectNameList.size());
        // 对文件名集合进行升序排序
        objectNameList.sort((o1, o2) -> Integer.parseInt(o2) > Integer.parseInt(o1) ? -1 : 1);
        //合并文件
        for (String object : objectNameList) {
            composeSourceList.add(ComposeSource.builder()
                    .bucket(originBucketName)
                    .object(object)
                    .build());
        }
        //上传合并文件
        return composeObject(composeSourceList, targetBucketName, objectName);
    }
}

1.3、yaml配置文件

minio:
  externalAccess: http://10.0.0.43:9010
  endpoint: http://10.0.0.43:9010
  accessKey: witminioadmin
  secretKey: witminioadmin
  bucketName: publicphospherus

二、工具类

2.1、IO

package com.worldintek.fms.utils;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;

/**
 * @author wxl
 * @version 1.0
 * @description: IO转换工具类
 * @date 2022/7/6 18:15
 */
public class IOConvertUtils {

    private IOConvertUtils() {
    }

    /**
     * inputStream转outputStream
     * **/
    public static ByteArrayOutputStream iConvertO(InputStream in) throws Exception {
        ByteArrayOutputStream swapStream = new ByteArrayOutputStream();
        int ch;
        while ((ch = in.read()) != -1) {
            swapStream.write(ch);
        }
        return swapStream;
    }

    /**
     * outputStream转inputStream
     * **/
    public static ByteArrayInputStream oConvertI(OutputStream out) throws Exception {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        baos = (ByteArrayOutputStream) out;
        return new ByteArrayInputStream(baos.toByteArray());
    }

    /**
     * inputStream转String
     * **/
    public static String iConvertString(InputStream in) throws Exception {
        ByteArrayOutputStream swapStream = new ByteArrayOutputStream();
        int ch;
        while ((ch = in.read()) != -1) {
            swapStream.write(ch);
        }
        return swapStream.toString();
    }

    /**
     * OutputStream 转String
     * **/
    public static String oToString(OutputStream out) throws Exception {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        baos = (ByteArrayOutputStream) out;
        ByteArrayInputStream swapStream = new ByteArrayInputStream(baos.toByteArray());
        return swapStream.toString();
    }

    /**
     * String转inputStream
     * **/
    public static ByteArrayInputStream stringConvertI(String in) throws Exception {
        return new ByteArrayInputStream(in.getBytes());
    }

    /**
     * String 转outputStream
     * **/
    public static ByteArrayOutputStream parse_outputStream(String in) throws Exception {
        return iConvertO(stringConvertI(in));
    }
}

2.2、分片

package com.worldintek.fms.utils;

import com.worldintek.fms.domain.MockMultipartFile;
import com.worldintek.fms.enumration.ShardFileStatusCode;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * @author wxl
 * @version 1.0
 * @description: 文件分片工具类
 * @date 2022/7/6 18:16
 */
@Slf4j
public class ShardFileUtils {

    private ShardFileUtils() {

    }


    /**
     * 大文件分片成 List<InputStream>
     *
     * @param file      文件路径；
     * @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
     * @return List<MultipartFile> 分片集合
     */
    @SneakyThrows
    public static List<InputStream> splitFileInputStreams(File file, long splitSize) {
        if (splitSize < (5 * 1024 * 1024)) {
            throw new Exception(ShardFileStatusCode.SHARD_MUST_MORE_THAN_5M.getMessage());
        }
        List<InputStream> inputStreams = new ArrayList<>();
        InputStream bis = null;//输入流用于读取文件数据
        OutputStream bos = null;//输出流用于输出分片文件至磁盘
        try {
            bis = new BufferedInputStream(new FileInputStream(file));
            long writeByte = 0;//已读取的字节数
            int len = 0;
            byte[] bt = new byte[5 * 1024 * 1024];
            while (-1 != (len = bis.read(bt))) {
                if (writeByte % splitSize == 0) {
                    bos = new ByteArrayOutputStream();
                }
                writeByte += len;
                bos.write(bt, 0, len);
                if (writeByte % splitSize == 0) {
                    InputStream inputStream = IOConvertUtils.oConvertI(bos);
                    inputStreams.add(inputStream);
                }
            }
            InputStream inputStream = IOConvertUtils.oConvertI(bos);
            inputStreams.add(inputStream);
            log.info("{} 文件分片成功！开始准备分片上传", file.getName());
        } catch (Exception e) {
            log.error("文件分片失败！原因：{}", e.getMessage());
            e.printStackTrace();
        }
        return inputStreams;
    }


    /**
     * Multipart文件分片成List<InputStream>
     *
     * @param multipartFile      MultipartFile
     * @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
     * @return List<MultipartFile> 分片集合
     */
    @SneakyThrows
    public static List<InputStream> splitMultipartFileInputStreams(MultipartFile multipartFile, long splitSize) {
        if (splitSize < (5 * 1024 * 1024)) {
            throw new Exception(ShardFileStatusCode.SHARD_MUST_MORE_THAN_5M.getMessage());
        }
        String filename = multipartFile.getOriginalFilename();
        List<InputStream> inputStreams = new ArrayList<>();
        InputStream bis = null;//输入流用于读取文件数据
        OutputStream bos = null;//输出流用于输出分片文件至磁盘
        try {
            bis = new BufferedInputStream(multipartFile.getInputStream());
            long writeByte = 0;//已读取的字节数
            int len = 0;
            byte[] bt = new byte[5 * 1024 * 1024];
            while (-1 != (len = bis.read(bt))) {
                if (writeByte % splitSize == 0) {
                    bos = new ByteArrayOutputStream();
                }
                writeByte += len;
                bos.write(bt, 0, len);
                if (writeByte % splitSize == 0) {
                    InputStream inputStream = IOConvertUtils.oConvertI(bos);
                    inputStreams.add(inputStream);
                }
            }
            InputStream inputStream = IOConvertUtils.oConvertI(bos);
            inputStreams.add(inputStream);
            log.info("{} 文件分片成功！", filename);
        } catch (Exception e) {
            log.error("文件分片失败！原因：{}", e.getMessage());
            e.printStackTrace();
        }
        return inputStreams;
    }

    /**
     * 大文件分片成List<MultipartFile>
     * @param file      文件路径
     * @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
     * @return List<MultipartFile>
     */
    public static List<MultipartFile> splitFileMultipartFiles(File file, long splitSize) {
        List<MultipartFile> files = new ArrayList<>();
        InputStream bis = null;//输入流用于读取文件数据
        OutputStream bos = null;//输出流用于输出分片文件至磁盘
        try {
            bis = new BufferedInputStream(new FileInputStream(file));
            long writeByte = 0;//已读取的字节数
            int len = 0;
            byte[] bt = new byte[5 * 1024 * 1024];
            while (-1 != (len = bis.read(bt))) {
                if (writeByte % splitSize == 0) {
                    bos = new ByteArrayOutputStream();
                }
                writeByte += len;
                bos.write(bt, 0, len);
                if (writeByte % splitSize == 0) {
                    InputStream inputStream = IOConvertUtils.oConvertI(bos);
                    MultipartFile multipartFile = new MockMultipartFile(String.valueOf((writeByte / splitSize)), inputStream);
                    files.add(multipartFile);
                }
            }
            InputStream inputStream = IOConvertUtils.oConvertI(bos);
            MultipartFile multipartFile = new MockMultipartFile(String.valueOf((writeByte / splitSize)), inputStream);
            files.add(multipartFile);
            System.out.println("文件分片成功！");
        } catch (Exception e) {
            System.out.println("文件分片失败！");
            e.printStackTrace();
        }
        return files;
    }

}

三、方法实现

3.1、分片

package com.worldintek.fms.service.impl;


import cn.hutool.core.util.StrUtil;
import com.worldintek.common.utils.sUUID;
import com.worldintek.fms.config.MinioConfig;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.worldintek.fms.domain.MergeShardArgs;
import com.worldintek.fms.domain.MinioTemplateResult;
import com.worldintek.fms.domain.ResponseEntry;
import com.worldintek.fms.entity.Md5FileNameEntry;
import com.worldintek.fms.enumration.ShardFileStatusCode;
import com.worldintek.fms.mapper.Md5FileNameMapper;
import com.worldintek.fms.service.MinioUploadFileService;
import com.worldintek.fms.service.SseEmitterService;
import com.worldintek.fms.template.MinioTemplate;
import com.worldintek.fms.utils.ShardFileUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ObjectUtils;
import org.springframework.web.multipart.MultipartFile;

import java.io.InputStream;
import java.util.*;
import java.util.concurrent.*;

/**
 * @author wxl
 * @version 1.0
 * @description: 实现类
 * @date 2022/7/8 13:33
 */
@Service
@Slf4j
public class MinioUploadFileServiceImpl implements MinioUploadFileService {


    @Autowired
    private MinioTemplate minioTemplate;

    @Autowired
    private MinioConfig minioConfig;

    @Autowired
    Md5FileNameMapper md5FileNameMapper;

    @Autowired
    SseEmitterService sseEmitterService;


    private static ExecutorService executorService;

    /**
     * @return ExecutorService 线程池
     * @description:因此IO密集型的任务，可大致设为： N(threads) = 2N(cpu) ->我的cpu是4核
     * @author wxl
     * @date 2022/7/20 23:41
     **/
    private static ExecutorService createThreadPool(int sizePool) {
        return new ThreadPoolExecutor(
                sizePool,
                sizePool,
                0L,
                TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>(1024));
    }


    /**
     * @param file     multipartFile
     * @param res      全局共享消息体
     * @param fileType 文件类型
     * @description: 大文件上传大于5MB分片文件上传
     * @author wxl
     * @date 2022/7/20 23:51
     **/
    @SneakyThrows
    @Override
    public Md5FileNameEntry uploadShard(MultipartFile file, ResponseEntry res, String fileType, String userUuid) {
        executorService = createThreadPool(8);
        // 上传过程中出现异常，状态码设置为50000
        boolean stopStatus = true;
        if (file == null) {
            res.get(userUuid).put("status", ShardFileStatusCode.FAILURE);
            throw new Exception(ShardFileStatusCode.FAILURE.getMessage());
        }
        Long fileSize = file.getSize();
        String md5BucketName = userUuid +"-"+ StrUtil.uuid();
        String fileName = file.getOriginalFilename();
        res.get(userUuid).put("md5BucketName", md5BucketName);
        //分片大小
        long shardSize = 5 * 1024 * 1024L;
        //开始分片
        List<InputStream> inputStreams = ShardFileUtils.splitMultipartFileInputStreams(file, shardSize);
        long shardCount = inputStreams.size(); //总片数
        //封装合并参数
        MergeShardArgs mergeShardArgs = new MergeShardArgs((int) shardCount, fileName, md5BucketName, fileType, fileSize);
        boolean fileExists = isFileExists(md5BucketName);
        boolean bucketExists = minioTemplate.bucketExists(md5BucketName);
        //当前文件不存在DB和minio  可以正常分片上传
        if (!fileExists && !bucketExists) {
            try {
                //创建临时桶
                minioTemplate.makeBucket(md5BucketName);
                uploadJob(shardCount, inputStreams, res, stopStatus, md5BucketName, fileName, userUuid);
                //开始合并
                Md5FileNameEntry md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
                //上传完成清空当前用户数据
                res.get(userUuid).clear();
                log.info("文件上传成功 {} ", file.getOriginalFilename());
                res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_SUCCESS.getCode());
                return md5FileNameEntry;
            } catch (Exception e) {
                log.error("分片合并失败：{}", e.getMessage());
                throw new Exception("分片合并失败");
            }
        }

        /*
          如果文件存在;
           1、存在DB minio == null 上传完成秒传
           2、存在minio  DB == null
          先看临时桶在不在
           1、在;断点上传
           2、在;没合并
          */
        else if (fileExists && !bucketExists) {
            //1、存在DB minio == null
            md5FileNameMapper.delete(new QueryWrapper<Md5FileNameEntry>()
                    .eq("md5_file_name", md5BucketName)
            );
            throw new Exception("请重新上传文件");

        } else if (!fileExists) {
            //2、存在minio  DB == null
//         *  1、在;断点上传
//         *  2、在;没合并
            List<String> objectNames = minioTemplate.listObjectNames(md5BucketName);
            Md5FileNameEntry md5FileNameEntry;
            //临时桶在; 断点上传
            if (objectNames.size() == shardCount) {
                //设置百分比
                res.get(userUuid).put("uploadPercent", 100);
                log.info("uploadPercent:{}", 100);
                //设置上传文件大小
                res.get(userUuid).put("uploadSize", fileSize);
                //没有合并: 合并秒传
                md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
                log.info("uploadSize：{}", fileSize);
                log.info("{} 秒传成功", fileName);
                //上传完成清空当前用户数据
                res.get(userUuid).clear();
            } else {
                //断点上传
                log.info("开始断点上传>>>>>>");
                List<String> containStr = containList(objectNames, shardCount);
                log.info("上传过的分片:" + containStr);
                CountDownLatch countDownLatch = new CountDownLatch(containStr.size());
                try {
                    log.info("开始断点分片上传:" + fileName);
                    for (String s : containStr) {
                        stopStatus = (boolean) res.get("userUuid").get("stopStatus");
                        if (stopStatus) {
                            int c = Integer.parseInt(s);
                            executorService.execute(new BranchThread(inputStreams.get(c - 1), md5BucketName, c, res, countDownLatch, shardCount, stopStatus, userUuid, minioTemplate, sseEmitterService));
                        } else {
                            executorService.shutdown();
                            break;
                        }
                    }
                } catch (Exception e) {
                    log.error("断点上传出现异常{}", e.getMessage());
                    throw new Exception("断点上传出现异常");
                } finally {
                    //关闭线程池
                    executorService.shutdown();
                }
                countDownLatch.await();
                log.info("所有分片上传完成");
                res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getCode());

                md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
                log.info("文件上传成功：{} ", fileName);
                //上传完成清空当前用户数据
                res.get(userUuid).clear();
            }
            res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_MERGE_SUCCESS.getCode());
            return md5FileNameEntry;

        } else {
            //出现异常
            log.error("出现异常: {}", ShardFileStatusCode.FOUND.getMessage());
            throw new Exception("文件上传出现异常");
        }
    }

    /**
     * @description:
     * @author wxl
     * @date 2022/7/22 22:49
     * @param mergeShardArgs 合并文件参数实体
     * @param userUuid user id
     * @return Md5FileNameEntry
     **/
    @Transactional
    @Override
    public Md5FileNameEntry mergeShard(MergeShardArgs mergeShardArgs, String userUuid, ResponseEntry res) {
        Integer shardCount = mergeShardArgs.getShardCount();
        String fileName = (String) res.get(userUuid).get("fileName");
        String md5 = mergeShardArgs.getMd5();
        try {
            List<String> objectNameList = minioTemplate.listObjectNames(md5);
            //查询的服务器的分片和传入的分片不同
            if (shardCount != objectNameList.size()) {
                // 失败
                log.error("服务器的分片{}和传入的分片不同{}", shardCount, objectNameList.size());
                throw new Exception("服务器的分片和传入的分片不同");
            } else {
                // 开始合并请求
                String targetBucketName = minioConfig.getBucketName();
                //拼接合并之后的文件名称
                String objectName = userUuid + "-" + fileName;
                //合并
                minioTemplate.composeObject(md5, targetBucketName, objectName);

                log.info("桶：{} 中的分片文件，已经在桶：{},文件 {} 合并成功", md5, targetBucketName, objectName);
                String url = minioTemplate.getPresignedObjectUrl(targetBucketName, objectName);
                // 合并成功之后删除对应的临时桶
                minioTemplate.removeBucket(md5, true);
                log.info("删除桶 {} 成功", md5);
                // 存入DB中
                Md5FileNameEntry md5FileNameEntry = new Md5FileNameEntry();
                md5FileNameEntry.setMd5FileName(md5);
                md5FileNameEntry.setUrl(url);
                md5FileNameEntry.setCreateTime(new Date());
                md5FileNameEntry.setUpdateTime(new Date());
                md5FileNameMapper.insert(md5FileNameEntry);
                log.info("文件合并成{}并存入DB", md5FileNameEntry);
                return md5FileNameEntry;
            }
        } catch (Exception e) {
            // 失败
            log.error("合并失败:{}", e.getMessage());
        }
        return null;

    }

    /**
     * 根据文件大小和文件的md5校验文件是否存在
     * 暂时使用Redis实现，后续需要存入数据库
     * 实现秒传接口
     *
     * @param md5 文件的md5
     * @return 操作是否成功
     */
    @SneakyThrows
    @Override
    public boolean isFileExists(String md5) {
        if (ObjectUtils.isEmpty(md5)) {
            log.error("参数md5为空");
            throw new Exception("参数md5为空");
        }
        // 查询
        Md5FileNameEntry md5FileNameEntry = md5FileNameMapper.selectOne(new QueryWrapper<Md5FileNameEntry>()
                .eq("md5_file_name", md5)
        );
        /*
          文件不存在 false
          文件存在 true
          */
        return md5FileNameEntry != null;
    }


    /**
     * @description: 小文件上传 0-5MB
     * @author wxl
     * @date 2022/7/22 22:48
     * @param multipartFile file
     * @param userUuid user id
     * @param voiceTimeSize 声音的长度
     * @return Md5FileNameEntry
     **/
    @Override
    public Md5FileNameEntry upload(MultipartFile multipartFile, String userUuid, ResponseEntry res, int voiceTimeSize) throws Exception {
        if (multipartFile == null) {
            throw new Exception(ShardFileStatusCode.FAILURE.getMessage());
        }
        if (userUuid == null) {
            throw new Exception(ShardFileStatusCode.FILE_IS_NULL.getMessage());
        }
        log.info(multipartFile.getName());
        String fileName = (String) res.get(userUuid).get("fileName");
        String targetBucketName = minioConfig.getBucketName();
        String targetName;
        if (voiceTimeSize <= 0){
            targetName = userUuid + "-" + fileName;
        }else {
            targetName = userUuid + "-" + voiceTimeSize + "-" +fileName;
        }
        if (isFileExists(targetName)){
            log.warn("File is exist already");
            throw new Exception("File is exist already");
        }
        else {
            MinioTemplateResult result = minioTemplate.putSimpleObject(multipartFile.getInputStream(), targetBucketName, targetName);
            log.info("小文件上传成功");
            String url = minioTemplate.getPresignedObjectUrl(targetBucketName, result.getOriginalFileName());
            Md5FileNameEntry md5FileNameEntry = new Md5FileNameEntry();
            md5FileNameEntry.setMd5FileName(result.getOriginalFileName());
            md5FileNameEntry.setUrl(url);
            md5FileNameEntry.setCreateTime(new Date());
            md5FileNameEntry.setUpdateTime(new Date());
            md5FileNameMapper.insert(md5FileNameEntry);
            log.info("小文件插入DB");
            return md5FileNameEntry;
        }

    }

    /**
     * @description: 0-6位取文件名
     * @author wxl
     * @date 2022/7/22 22:50
     * @param oleName  oleName
     * @return String
     **/
    @Override
    public String rename(String oleName) {
        return oleName.substring(0, 6);
    }

    /**
     * @param objNames   the object names
     * @param shardCount the shard count
     * @return List<String>
     * @description: 查询上传过的分片
     * @author wxl
     * @date 2022/7/21 0:00
     **/
    private List<String> containList(List<String> objNames, long shardCount) {
        List<String> containList = new ArrayList<>();
        for (int i = 1; i <= shardCount; i++) {
            String str = String.valueOf(i);
            if (!objNames.contains(str)) {
                containList.add(str);
            }
        }
        return containList;
    }


    /**
     * @param shardCount    number of shards
     * @param inputStreams  files stream
     * @param res           全局共享消息体
     * @param stopStatus    暂停状态
     * @param md5BucketName md5 bucket name
     * @param fileName      file name
     * @description:分片任务
     * @author wxl
     * @date 2022/7/20 23:54
     **/
    private void uploadJob(long shardCount, List<InputStream> inputStreams, ResponseEntry res, Boolean stopStatus, String md5BucketName, String fileName, String userUuid) throws InterruptedException {
        CountDownLatch countDownLatch = new CountDownLatch((int) shardCount);
        if (shardCount > 10000) {
            throw new RuntimeException("Total parts count should not exceed 10000");
        }
        log.info("文件分得总片数：" + shardCount);
        try {
            log.info("开始分片上传:" + fileName);
            for (int i = 0; i < shardCount; i++) {

                stopStatus = (Boolean) res.get(userUuid).get("stopStatus");
                if (stopStatus) {
                    executorService.execute(new BranchThread(inputStreams.get(i), md5BucketName, i + 1, res, countDownLatch, shardCount, stopStatus, userUuid, minioTemplate, sseEmitterService));
                } else {
                    executorService.shutdown();
                    break;
                }
            }
        } catch (Exception e) {
            log.error("线程上传出现异常:{}", e.getMessage());
        } finally {
            //关闭线程池
            executorService.shutdown();
        }
        log.info(">>>>>>>>>>等待分片上传");
        countDownLatch.await();

        log.info(">>>>>>>>>>所有分片上传完成");
        res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getCode());
    }

    /**
     * @author wxl
     * @date 2022/7/20 23:57
     * @description: 分片上传内部类
     **/
    private static class BranchThread implements Runnable {
        /**
         * 文件流
         */
        private final InputStream inputStream;

        /**
         * md5名
         */
        private final String md5BucketName;

        /**
         * 当前片数
         */
        private final Integer curIndex;

        /**
         * 返回给前端的res
         */
        private final ResponseEntry res;

        /**
         * 计数等待线程执行完成
         */
        private final CountDownLatch countDownLatch;

        /**
         * 总片数
         */
        private final long shardCount;

        /**
         * 暂停状态
         */
        private final boolean stopStatus;

        /**
         * 用户id
         */
        private final String userUuid;


        /**
         * template
         */
        private final MinioTemplate minioTemplate;
        /**
         * sse发给前端的服务
         */
        private final SseEmitterService sseEmitterService;

        public BranchThread(InputStream inputStream, String md5BucketName, Integer curIndex, ResponseEntry res, CountDownLatch countDownLatch, long shardCount, boolean stopStatus, String userUuid, MinioTemplate minioTemplate, SseEmitterService sseEmitterService) {
            this.inputStream = inputStream;
            this.md5BucketName = md5BucketName;
            this.curIndex = curIndex;
            this.res = res;
            this.countDownLatch = countDownLatch;
            this.shardCount = shardCount;
            this.stopStatus = stopStatus;
            this.userUuid = userUuid;
            this.minioTemplate = minioTemplate;
            this.sseEmitterService = sseEmitterService;
        }


        @SneakyThrows
        @Override
        public void run() {
            try {
                if (stopStatus) {
                    Long uploadPercent = ((curIndex * 100) / shardCount);
                    String curIndexName = String.valueOf(curIndex);
                    //设置百分比
                    res.get(userUuid).put("uploadPercent", uploadPercent);
                    log.info("uploadPercent:{}", uploadPercent);
                    //设置上传文件大小
                    res.get(userUuid).put("uploadSize", inputStream.available());
                    log.info("uploadSize：{}", inputStream.available());
//                    sseEmitterService.sendResMapToOneClient(userUuid, res);
                    MinioTemplateResult minioTemplateResult = minioTemplate.putChunkObject(inputStream, md5BucketName, curIndexName);
                    log.info("分片上传成功： {}", minioTemplateResult);
                } else {
                    executorService.shutdown();
                }

            } catch (Exception e) {
                log.error("线程上传分片异常：{}", e.getMessage());
            } finally {
                countDownLatch.countDown();
            }
        }
    }
}

3.2、SSE进度消息推送

package com.worldintek.fms.service.impl;

import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpStatus;
import com.worldintek.common.exception.ApiException;
import com.worldintek.fms.domain.MessageVo;
import com.worldintek.fms.domain.ResponseEntry;
import com.worldintek.fms.service.SseEmitterService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;

import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;

/**
 * @author wxl
 * @version 1.0
 * @description: SseEmitter发送信息的服务类
 * @date 2022/7/8 13:37
 */
@Service
@Slf4j
public class SseEmitterServiceImpl extends SseEmitter implements SseEmitterService {

    @Autowired
    private ResponseEntry res;
    /**
     * 容器，保存连接，用于输出返回 ;可使用其他方法实现
     */
    private static final Map<String, SseEmitter> sseCache = new ConcurrentHashMap<>();



    /**
     * 根据客户端id获取SseEmitter对象
     *
     * @param clientId 客户端ID
     */
    @Override
    public SseEmitter getSseEmitterByClientId(String clientId) {
        return sseCache.get(clientId);
    }




    /**
     * 创建连接
     *
     * @param clientId 客户端ID
     */
    @Override
    public SseEmitter createConnect(String clientId) {
        if (clientId == null) {
            throw new ApiException("用户Id为不能建立连接");
        }
        // 设置超时时间，0表示不过期。默认30秒，超过时间未完成会抛出异常：AsyncRequestTimeoutException
        SseEmitter sseEmitter = new SseEmitter(100000L);
        // 是否需要给客户端推送ID
//        if (StrUtil.isBlank(clientId)) {
//            clientId = IdUtil.simpleUUID();
//            res.put(clientId, Map.of("", clientId));
//        }
//        res.put("clientId", Map.of("clientId", clientId));
        log.info("当前clientId----->{}", clientId);

        // 注册回调
        // 长链接完成后回调接口(即关闭连接时调用)
        sseEmitter.onCompletion(completionCallBack(clientId));
        // 连接超时回调
        sseEmitter.onTimeout(timeoutCallBack(clientId));
        // 推送消息异常时，回调方法
        sseEmitter.onError(errorCallBack(clientId));
        sseCache.put(clientId, sseEmitter);
        log.info("创建新的sse连接，当前用户：{}    累计用户:{}", clientId, sseCache.size());
        try {
//             注册成功返回用户信息
            sseEmitter.send(SseEmitter.event()
                    .id(clientId)
                    .data("HttpStatus----->" + HttpStatus.HTTP_CREATED)
            );
        } catch (IOException e) {
            log.error("创建长链接异常，客户端ID:{}   异常信息:{}", clientId, e.getMessage());
        }
        return sseEmitter;
    }


    /**
     * 发送消息给所有客户端
     *
     * @param msg 消息内容
     */
    @Override
    public void sendMessageToAllClient(String msg) {
        if (MapUtil.isEmpty(sseCache)) {
            return;
        }
        // 判断发送的消息是否为空
        for (Map.Entry<String, SseEmitter> entry : sseCache.entrySet()) {
            MessageVo messageVo = new MessageVo();
            messageVo.setClientId(entry.getKey());
            messageVo.setData(msg);
            sendMsgToClientByClientId(entry.getKey(), messageVo, entry.getValue());
        }

    }

    /**
     * 给指定客户端发送消息
     *
     * @param clientId 客户端ID
     * @param msg      消息内容
     */
    @Override
    public void sendMessageToOneClient(String clientId, String msg) {
        MessageVo messageVo = new MessageVo(clientId, msg);
        sendMsgToClientByClientId(clientId, messageVo, sseCache.get(clientId));
    }

    /**
     * 给指定客户端发送消息
     *
     * @param clientId 客户端ID
     * @param responseEntry      消息内容
     */
    @Override
    public void sendResMapToOneClient(String clientId, ResponseEntry responseEntry) {
        sendResMapToClientByClientId(clientId, responseEntry, sseCache.get(clientId));
    }


    /**
     * @description: 服务器给客户端发送消息
     * @author wxl
     * @date 2022/7/21 16:00
     * @param clientId 客户端ID
     **/
    @Override
    public void sendMessage(String clientId)  {
        sendResMapToOneClient(clientId, res);
    }

    /**
     * 推送消息到客户端
     * 此处做了推送失败后，重试推送机制，可根据自己业务进行修改
     *
     * @param clientId  客户端ID
     * @param responseEntry 推送信息，此处结合具体业务，定义自己的返回值即可
     **/
    private void sendResMapToClientByClientId(String clientId, ResponseEntry responseEntry, SseEmitter sseEmitter) {
        if (sseEmitter == null) {
            log.error("推送消息失败：客户端{}未创建长链接,失败消息:{}",
                    clientId, responseEntry.toString());
            return;
        }
        SseEventBuilder sendData = SseEmitter.event().data("HttpStatus:" + HttpStatus.HTTP_OK, MediaType.APPLICATION_JSON)
                .data(responseEntry, MediaType.APPLICATION_JSON);
        try {
            sseEmitter.send(sendData);
        } catch (IOException e) {
            // 推送消息失败，记录错误日志，进行重推
            log.error("推送消息失败：{},尝试进行重推", responseEntry.toString());
            boolean isSuccess = true;
            // 推送消息失败后，每隔10s推送一次，推送5次
            for (int i = 0; i < 5; i++) {
                try {
                    Thread.sleep(10000);
                    sseEmitter = sseCache.get(clientId);
                    if (sseEmitter == null) {
                        log.error("{}的第{}次消息重推失败，未创建长链接", clientId, i + 1);
                        continue;
                    }
                    sseEmitter.send(sendData);
                } catch (Exception ex) {
                    log.error("{}的第{}次消息重推失败", clientId, i + 1, ex);
                    continue;
                }
                log.info("{}的第{}次消息重推成功,{}", clientId, i + 1, responseEntry.toString());
                return;
            }
        }
    }


    /**
     * 推送消息到客户端
     * 此处做了推送失败后，重试推送机制，可根据自己业务进行修改
     *
     * @param clientId  客户端ID
     * @param messageVo 推送信息，此处结合具体业务，定义自己的返回值即可
     **/
    private void sendMsgToClientByClientId(String clientId, MessageVo messageVo, SseEmitter sseEmitter) {
        if (sseEmitter == null) {
            log.error("推送消息失败：客户端{}未创建长链接,失败消息:{}",
                    clientId, messageVo.toString());
            return;
        }
        SseEventBuilder sendData = SseEmitter.event().data("HttpStatus:" + HttpStatus.HTTP_OK, MediaType.APPLICATION_JSON)
                .data(messageVo, MediaType.APPLICATION_JSON);
        try {
            sseEmitter.send(sendData);
        } catch (IOException e) {
            // 推送消息失败，记录错误日志，进行重推
            log.error("推送消息失败：{},尝试进行重推", messageVo.toString());
            boolean isSuccess = true;
            // 推送消息失败后，每隔10s推送一次，推送5次
            for (int i = 0; i < 5; i++) {
                try {
                    Thread.sleep(10000);
                    sseEmitter = sseCache.get(clientId);
                    if (sseEmitter == null) {
                        log.error("{}的第{}次消息重推失败，未创建长链接", clientId, i + 1);
                        continue;
                    }
                    sseEmitter.send(sendData);
                } catch (Exception ex) {
                    log.error("{}的第{}次消息重推失败", clientId, i + 1, ex);
                    continue;
                }
                log.info("{}的第{}次消息重推成功,{}", clientId, i + 1, messageVo.toString());
                return;
            }
        }
    }


    /**
     * 关闭连接
     *
     * @param clientId 客户端ID
     */
    @Override
    public void closeConnect(String clientId) {
        SseEmitter sseEmitter = sseCache.get(clientId);
        if (sseEmitter != null) {
            sseEmitter.complete();
            removeUser(clientId);
        }
    }



    /**
     * 长链接完成后回调接口(即关闭连接时调用)
     *
     * @param clientId 客户端ID
     **/
    private Runnable completionCallBack(String clientId) {
        return () -> {
            log.info("结束连接：{}", clientId);
            removeUser(clientId);
        };
    }

    /**
     * 连接超时时调用
     *
     * @param clientId 客户端ID
     **/
    private Runnable timeoutCallBack(String clientId) {
        return () -> {
            log.info("连接超时：{}", clientId);
            removeUser(clientId);
        };
    }

    /**
     * 推送消息异常时，回调方法
     *
     * @param clientId 客户端ID
     **/
    private Consumer<Throwable> errorCallBack(String clientId) {
        return throwable -> {
            log.error("SseEmitterServiceImpl[errorCallBack]：连接异常,客户端ID:{}", clientId);

            // 推送消息失败后，每隔10s推送一次，推送5次
            for (int i = 0; i < 5; i++) {
                try {
                    Thread.sleep(10000);
                    SseEmitter sseEmitter = sseCache.get(clientId);
                    if (sseEmitter == null) {
                        log.error("SseEmitterServiceImpl[errorCallBack]：第{}次消息重推失败,未获取到 {} 对应的长链接", i + 1, clientId);
                        continue;
                    }
                    sseEmitter.send("失败后重新推送");
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        };
    }

    /**
     * 移除用户连接
     *
     * @param clientId 客户端ID
     **/
    private void removeUser(String clientId) {
        sseCache.remove(clientId);
        log.info("SseEmitterServiceImpl[removeUser]:移除用户：{}", clientId);
    }
}