读取须知:需掌握minio、多线程、SSE的使用
一、minio配置
1、minio配置
1.1、配置
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.worldintek.fms.template.MinioTemplate;
import io.minio.MinioClient;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
@ConfigurationProperties(value = "minio")
public class MinioConfig {
/**
* 对象存储服务的对外URL
*/
private String externalAccess;
/**
* 对象存储服务的URL
*/
private String endpoint;
/**
* Access key就像用户ID,可以唯一标识你的账户。
*/
private String accessKey;
/**
* Secret key是你账户的密码。
*/
private String secretKey;
/**
* bucketName是你设置的桶的名称
*/
private String bucketName;
/**
* 初始化一个MinIO客户端用来连接MinIO存储服务
*
* @return MinioClient
*/
@Bean(name = "minioClient")
public MinioClient initMinioClient() {
return MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build();
}
/**
* 初始化MinioTemplate,封装了一些MinIOClient的基本操作
*
* @return MinioTemplate
*/
@Bean(name = "minioTemplate")
public MinioTemplate minioTemplate() {
return new MinioTemplate(initMinioClient(), this);
}
@JsonIgnore
public String getPublicUrlPrefix(){
return externalAccess + "/" + bucketName;
}
}
1.2、模板
package com.worldintek.fms.template;
import com.worldintek.fms.config.MinioConfig;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import com.worldintek.fms.domain.MinioTemplateResult;
import io.minio.*;
import io.minio.http.Method;
import io.minio.messages.Bucket;
import io.minio.messages.Item;
import lombok.AllArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import javax.annotation.PostConstruct;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
* @author wxl
* @description:
* @version: v1.0
* @since 2022-05-03 13:04
*/
@Slf4j
@AllArgsConstructor
public class MinioTemplate {
/**
* MinIO 客户端
*/
private final MinioClient minioClient;
/**
* MinIO 配置类
*/
private final MinioConfig minioConfig;
/**
* 查询所有存储桶
*
* @return Bucket 集合
*/
@SneakyThrows
public List<Bucket> listBuckets() {
return minioClient.listBuckets();
}
/**
* 桶是否存在
*
* @param bucketName 桶名
* @return 是否存在
*/
@SneakyThrows
public boolean bucketExists(String bucketName) {
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
}
/**
* 创建存储桶
*
* @param bucketName 桶名
*/
@SneakyThrows
public void makeBucket(String bucketName) {
if (!bucketExists(bucketName)) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
}
}
/**
* 删除一个空桶 如果存储桶存在对象不为空时,删除会报错。
*
* @param bucketName 桶名
*/
@SneakyThrows
public void removeBucket(String bucketName) {
removeBucket(bucketName, false);
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
}
/**
* 删除一个桶 根据桶是否存在数据进行不同的删除
* 桶为空时直接删除
* 桶不为空时先删除桶中的数据,然后再删除桶
*
* @param bucketName 桶名
*/
@SneakyThrows
public void removeBucket(String bucketName, boolean bucketNotNull) {
if (bucketNotNull) {
deleteBucketAllObject(bucketName);
}
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
}
/**
* @description: 创建文件夹
* @author wxl
* @date 2022/7/21 17:13
* @param
* @return
**/
@SneakyThrows
public MinioTemplateResult createDir(String dirName, String bucketName){
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucketName)
.stream(new ByteArrayInputStream(new byte[] {}), 0, -1)
.object(dirName + "/")
.build());
return new MinioTemplateResult(dirName, null);
}
/**
* 上传文件
*
* @param inputStream 流
* @param originalFileName 原始文件名
* @param bucketName 桶名
* @return ObjectWriteResponse
*/
@SneakyThrows
public MinioTemplateResult putSimpleObject(InputStream inputStream, String bucketName, String originalFileName) {
try {
if (ObjectUtils.isEmpty(bucketName)) {
bucketName = minioConfig.getBucketName();
}
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucketName)
.object(originalFileName)
.stream(inputStream, inputStream.available(), -1)
.build());
return new MinioTemplateResult(originalFileName, originalFileName);
} finally {
if (inputStream != null) {
inputStream.close();
}
}
}
/**
* 上传文件
*
* @param inputStream 流
* @param originalFileName 原始文件名
* @param bucketName 桶名
* @return ObjectWriteResponse
*/
@SneakyThrows
public MinioTemplateResult putObject(InputStream inputStream, String bucketName, String originalFileName) {
String uuidFileName = generateFileInMinioName(originalFileName);
try {
if (ObjectUtils.isEmpty(bucketName)) {
bucketName = minioConfig.getBucketName();
}
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucketName)
.object(uuidFileName)
.stream(inputStream, inputStream.available(), -1)
.build());
return new MinioTemplateResult(uuidFileName, originalFileName);
} finally {
if (inputStream != null) {
inputStream.close();
}
}
}
/**
* 删除桶中所有的对象
*
* @param bucketName 桶对象
*/
@SneakyThrows
public void deleteBucketAllObject(String bucketName) {
List<String> list = listObjectNames(bucketName);
if (!list.isEmpty()) {
for (String objectName : list) {
deleteObject(bucketName, objectName);
}
}
}
/**
* 查询桶中所有的对象名
*
* @param bucketName 桶名
* @return objectNames
*/
@SneakyThrows
public List<String> listObjectNames(String bucketName) {
List<String> objectNameList = new ArrayList<>();
if (bucketExists(bucketName)) {
Iterable<Result<Item>> results = listObjects(bucketName, true);
for (Result<Item> result : results) {
String objectName = result.get().objectName();
objectNameList.add(objectName);
}
}
return objectNameList;
}
/**
* 删除一个对象
*
* @param bucketName 桶名
* @param objectName 对象名
*/
@SneakyThrows
public void deleteObject(String bucketName, String objectName) {
minioClient.removeObject(RemoveObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.build());
}
/**
* 上传分片文件
*
* @param inputStream 流
* @param objectName 存入桶中的对象名
* @param bucketName 桶名
* @return ObjectWriteResponse
*/
@SneakyThrows
public MinioTemplateResult putChunkObject(InputStream inputStream, String bucketName, String objectName) {
try {
minioClient.putObject(
PutObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.stream(inputStream, inputStream.available(), -1)
.build());
return new MinioTemplateResult(objectName, objectName);
} finally {
if (inputStream != null) {
inputStream.close();
}
}
}
/**
* 返回临时带签名、Get请求方式的访问URL
*
* @param bucketName 桶名
* @param filePath Oss文件路径
* @return 临时带签名、Get请求方式的访问URL
*/
@SneakyThrows
public String getPresignedObjectUrl(String bucketName, String filePath) {
return minioClient.getPresignedObjectUrl(
GetPresignedObjectUrlArgs.builder()
.method(Method.GET)
.bucket(bucketName)
.object(filePath)
.build());
}
/**
* 返回临时带签名、过期时间为1天的PUT请求方式的访问URL
*
* @param bucketName 桶名
* @param filePath Oss文件路径
* @param queryParams 查询参数
* @return 临时带签名、过期时间为1天的PUT请求方式的访问URL
*/
@SneakyThrows
public String getPresignedObjectUrl(String bucketName, String filePath, Map<String, String> queryParams) {
return minioClient.getPresignedObjectUrl(
GetPresignedObjectUrlArgs.builder()
.method(Method.PUT)
.bucket(bucketName)
.object(filePath)
.expiry(1, TimeUnit.DAYS)
.extraQueryParams(queryParams)
.build());
}
/**
* GetObject接口用于获取某个文件(Object)。此操作需要对此Object具有读权限。
*
* @param bucketName 桶名
* @param objectName 文件路径
*/
@SneakyThrows
public InputStream getObject(String bucketName, String objectName) {
return minioClient.getObject(
GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
/**
* 查询桶的对象信息
*
* @param bucketName 桶名
* @param recursive 是否递归查询
* @return 桶的对象信息
*/
@SneakyThrows
public Iterable<Result<Item>> listObjects(String bucketName, boolean recursive) {
return minioClient.listObjects(
ListObjectsArgs.builder().bucket(bucketName).recursive(recursive).build());
}
/**
* 获取带签名的临时上传元数据对象,前端可获取后,直接上传到Minio
*
* @param bucketName 桶名称
* @param fileName 文件名
* @return Map<String, String>
*/
@SneakyThrows
public Map<String, String> getPresignedPostFormData(String bucketName, String fileName) {
// 为存储桶创建一个上传策略,过期时间为7天
PostPolicy policy = new PostPolicy(bucketName, ZonedDateTime.now().plusDays(1));
// 设置一个参数key,值为上传对象的名称
policy.addEqualsCondition("key", fileName);
// 添加Content-Type,例如以"image/"开头,表示只能上传照片,这里吃吃所有
policy.addStartsWithCondition("Content-Type", MediaType.ALL_VALUE);
// 设置上传文件的大小 64kiB to 10MiB.
//policy.addContentLengthRangeCondition(64 * 1024, 10 * 1024 * 1024);
return minioClient.getPresignedPostFormData(policy);
}
public String generateFileInMinioName(String originalFilename) {
return "files" + StrUtil.SLASH + DateUtil.format(new Date(), "yyyy-MM-dd") + StrUtil.SLASH + UUID.randomUUID() + StrUtil.UNDERLINE + originalFilename;
}
/**
* 初始化默认存储桶
*/
@PostConstruct
public void initDefaultBucket() {
String defaultBucketName = minioConfig.getBucketName();
if (bucketExists(defaultBucketName)) {
log.info("默认存储桶:{}已存在", defaultBucketName);
} else {
log.info("创建默认存储桶:{}", defaultBucketName);
makeBucket(minioConfig.getBucketName());
}
;
}
/**
* 文件合并,将分块文件组成一个新的文件
*
* @param bucketName 合并文件生成文件所在的桶
* @param fileName 原始文件名
* @param sourceObjectList 分块文件集合
* @return MinioTemplateResult
*/
@SneakyThrows
public MinioTemplateResult composeObject(String bucketName, String fileName, List<ComposeSource> sourceObjectList) {
String filenameExtension = StringUtils.getFilenameExtension(fileName);
String objectName = UUID.randomUUID() + "." + filenameExtension;
minioClient.composeObject(ComposeObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.sources(sourceObjectList)
.build());
String presignedObjectUrl = getPresignedObjectUrl(bucketName, fileName);
return new MinioTemplateResult(presignedObjectUrl, fileName);
}
/**
* 文件合并,将分块文件组成一个新的文件
*
* @param bucketName 合并文件生成文件所在的桶
* @param objectName 原始文件名
* @param sourceObjectList 分块文件集合
* @return MinioTemplateResult
*/
@SneakyThrows
public MinioTemplateResult composeObject(List<ComposeSource> sourceObjectList, String bucketName, String objectName) {
minioClient.composeObject(ComposeObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.sources(sourceObjectList)
.build());
String presignedObjectUrl = getPresignedObjectUrl(bucketName, objectName);
return new MinioTemplateResult(presignedObjectUrl, objectName);
}
/**
* 文件合并,将分块文件组成一个新的文件
*
* @param originBucketName 分块文件所在的桶
* @param targetBucketName 合并文件生成文件所在的桶
* @param objectName 存储于桶中的对象名
* @return MinioTemplateResult
*/
@SneakyThrows
public MinioTemplateResult composeObject(String originBucketName, String targetBucketName, String objectName) {
Iterable<Result<Item>> results = listObjects(originBucketName, true);
List<String> objectNameList = new ArrayList<>();
for (Result<Item> result : results) {
Item item = result.get();
objectNameList.add(item.objectName());
}
if (ObjectUtils.isEmpty(objectNameList)) {
throw new IllegalArgumentException(originBucketName + "桶中没有文件,请检查");
}
List<ComposeSource> composeSourceList = new ArrayList<>(objectNameList.size());
// 对文件名集合进行升序排序
objectNameList.sort((o1, o2) -> Integer.parseInt(o2) > Integer.parseInt(o1) ? -1 : 1);
//合并文件
for (String object : objectNameList) {
composeSourceList.add(ComposeSource.builder()
.bucket(originBucketName)
.object(object)
.build());
}
//上传合并文件
return composeObject(composeSourceList, targetBucketName, objectName);
}
}
1.3、yaml配置文件
minio:
externalAccess: http://10.0.0.43:9010
endpoint: http://10.0.0.43:9010
accessKey: witminioadmin
secretKey: witminioadmin
bucketName: publicphospherus
二、工具类
2.1、IO
package com.worldintek.fms.utils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
/**
* @author wxl
* @version 1.0
* @description: IO转换工具类
* @date 2022/7/6 18:15
*/
public class IOConvertUtils {
private IOConvertUtils() {
}
/**
* inputStream转outputStream
* **/
public static ByteArrayOutputStream iConvertO(InputStream in) throws Exception {
ByteArrayOutputStream swapStream = new ByteArrayOutputStream();
int ch;
while ((ch = in.read()) != -1) {
swapStream.write(ch);
}
return swapStream;
}
/**
* outputStream转inputStream
* **/
public static ByteArrayInputStream oConvertI(OutputStream out) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos = (ByteArrayOutputStream) out;
return new ByteArrayInputStream(baos.toByteArray());
}
/**
* inputStream转String
* **/
public static String iConvertString(InputStream in) throws Exception {
ByteArrayOutputStream swapStream = new ByteArrayOutputStream();
int ch;
while ((ch = in.read()) != -1) {
swapStream.write(ch);
}
return swapStream.toString();
}
/**
* OutputStream 转String
* **/
public static String oToString(OutputStream out) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos = (ByteArrayOutputStream) out;
ByteArrayInputStream swapStream = new ByteArrayInputStream(baos.toByteArray());
return swapStream.toString();
}
/**
* String转inputStream
* **/
public static ByteArrayInputStream stringConvertI(String in) throws Exception {
return new ByteArrayInputStream(in.getBytes());
}
/**
* String 转outputStream
* **/
public static ByteArrayOutputStream parse_outputStream(String in) throws Exception {
return iConvertO(stringConvertI(in));
}
}
2.2、分片
package com.worldintek.fms.utils;
import com.worldintek.fms.domain.MockMultipartFile;
import com.worldintek.fms.enumration.ShardFileStatusCode;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author wxl
* @version 1.0
* @description: 文件分片工具类
* @date 2022/7/6 18:16
*/
@Slf4j
public class ShardFileUtils {
private ShardFileUtils() {
}
/**
* 大文件分片成 List<InputStream>
*
* @param file 文件路径;
* @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
* @return List<MultipartFile> 分片集合
*/
@SneakyThrows
public static List<InputStream> splitFileInputStreams(File file, long splitSize) {
if (splitSize < (5 * 1024 * 1024)) {
throw new Exception(ShardFileStatusCode.SHARD_MUST_MORE_THAN_5M.getMessage());
}
List<InputStream> inputStreams = new ArrayList<>();
InputStream bis = null;//输入流用于读取文件数据
OutputStream bos = null;//输出流用于输出分片文件至磁盘
try {
bis = new BufferedInputStream(new FileInputStream(file));
long writeByte = 0;//已读取的字节数
int len = 0;
byte[] bt = new byte[5 * 1024 * 1024];
while (-1 != (len = bis.read(bt))) {
if (writeByte % splitSize == 0) {
bos = new ByteArrayOutputStream();
}
writeByte += len;
bos.write(bt, 0, len);
if (writeByte % splitSize == 0) {
InputStream inputStream = IOConvertUtils.oConvertI(bos);
inputStreams.add(inputStream);
}
}
InputStream inputStream = IOConvertUtils.oConvertI(bos);
inputStreams.add(inputStream);
log.info("{} 文件分片成功!开始准备分片上传", file.getName());
} catch (Exception e) {
log.error("文件分片失败!原因:{}", e.getMessage());
e.printStackTrace();
}
return inputStreams;
}
/**
* Multipart文件分片成List<InputStream>
*
* @param multipartFile MultipartFile
* @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
* @return List<MultipartFile> 分片集合
*/
@SneakyThrows
public static List<InputStream> splitMultipartFileInputStreams(MultipartFile multipartFile, long splitSize) {
if (splitSize < (5 * 1024 * 1024)) {
throw new Exception(ShardFileStatusCode.SHARD_MUST_MORE_THAN_5M.getMessage());
}
String filename = multipartFile.getOriginalFilename();
List<InputStream> inputStreams = new ArrayList<>();
InputStream bis = null;//输入流用于读取文件数据
OutputStream bos = null;//输出流用于输出分片文件至磁盘
try {
bis = new BufferedInputStream(multipartFile.getInputStream());
long writeByte = 0;//已读取的字节数
int len = 0;
byte[] bt = new byte[5 * 1024 * 1024];
while (-1 != (len = bis.read(bt))) {
if (writeByte % splitSize == 0) {
bos = new ByteArrayOutputStream();
}
writeByte += len;
bos.write(bt, 0, len);
if (writeByte % splitSize == 0) {
InputStream inputStream = IOConvertUtils.oConvertI(bos);
inputStreams.add(inputStream);
}
}
InputStream inputStream = IOConvertUtils.oConvertI(bos);
inputStreams.add(inputStream);
log.info("{} 文件分片成功!", filename);
} catch (Exception e) {
log.error("文件分片失败!原因:{}", e.getMessage());
e.printStackTrace();
}
return inputStreams;
}
/**
* 大文件分片成List<MultipartFile>
* @param file 文件路径
* @param splitSize = 5 * 1024 * 1024;//单片文件大小,5M
* @return List<MultipartFile>
*/
public static List<MultipartFile> splitFileMultipartFiles(File file, long splitSize) {
List<MultipartFile> files = new ArrayList<>();
InputStream bis = null;//输入流用于读取文件数据
OutputStream bos = null;//输出流用于输出分片文件至磁盘
try {
bis = new BufferedInputStream(new FileInputStream(file));
long writeByte = 0;//已读取的字节数
int len = 0;
byte[] bt = new byte[5 * 1024 * 1024];
while (-1 != (len = bis.read(bt))) {
if (writeByte % splitSize == 0) {
bos = new ByteArrayOutputStream();
}
writeByte += len;
bos.write(bt, 0, len);
if (writeByte % splitSize == 0) {
InputStream inputStream = IOConvertUtils.oConvertI(bos);
MultipartFile multipartFile = new MockMultipartFile(String.valueOf((writeByte / splitSize)), inputStream);
files.add(multipartFile);
}
}
InputStream inputStream = IOConvertUtils.oConvertI(bos);
MultipartFile multipartFile = new MockMultipartFile(String.valueOf((writeByte / splitSize)), inputStream);
files.add(multipartFile);
System.out.println("文件分片成功!");
} catch (Exception e) {
System.out.println("文件分片失败!");
e.printStackTrace();
}
return files;
}
}
三、方法实现
3.1、分片
package com.worldintek.fms.service.impl;
import cn.hutool.core.util.StrUtil;
import com.worldintek.common.utils.sUUID;
import com.worldintek.fms.config.MinioConfig;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.worldintek.fms.domain.MergeShardArgs;
import com.worldintek.fms.domain.MinioTemplateResult;
import com.worldintek.fms.domain.ResponseEntry;
import com.worldintek.fms.entity.Md5FileNameEntry;
import com.worldintek.fms.enumration.ShardFileStatusCode;
import com.worldintek.fms.mapper.Md5FileNameMapper;
import com.worldintek.fms.service.MinioUploadFileService;
import com.worldintek.fms.service.SseEmitterService;
import com.worldintek.fms.template.MinioTemplate;
import com.worldintek.fms.utils.ShardFileUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ObjectUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.*;
/**
* @author wxl
* @version 1.0
* @description: 实现类
* @date 2022/7/8 13:33
*/
@Service
@Slf4j
public class MinioUploadFileServiceImpl implements MinioUploadFileService {
@Autowired
private MinioTemplate minioTemplate;
@Autowired
private MinioConfig minioConfig;
@Autowired
Md5FileNameMapper md5FileNameMapper;
@Autowired
SseEmitterService sseEmitterService;
private static ExecutorService executorService;
/**
* @return ExecutorService 线程池
* @description:因此IO密集型的任务,可大致设为: N(threads) = 2N(cpu) ->我的cpu是4核
* @author wxl
* @date 2022/7/20 23:41
**/
private static ExecutorService createThreadPool(int sizePool) {
return new ThreadPoolExecutor(
sizePool,
sizePool,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(1024));
}
/**
* @param file multipartFile
* @param res 全局共享消息体
* @param fileType 文件类型
* @description: 大文件上传大于5MB分片文件上传
* @author wxl
* @date 2022/7/20 23:51
**/
@SneakyThrows
@Override
public Md5FileNameEntry uploadShard(MultipartFile file, ResponseEntry res, String fileType, String userUuid) {
executorService = createThreadPool(8);
// 上传过程中出现异常,状态码设置为50000
boolean stopStatus = true;
if (file == null) {
res.get(userUuid).put("status", ShardFileStatusCode.FAILURE);
throw new Exception(ShardFileStatusCode.FAILURE.getMessage());
}
Long fileSize = file.getSize();
String md5BucketName = userUuid +"-"+ StrUtil.uuid();
String fileName = file.getOriginalFilename();
res.get(userUuid).put("md5BucketName", md5BucketName);
//分片大小
long shardSize = 5 * 1024 * 1024L;
//开始分片
List<InputStream> inputStreams = ShardFileUtils.splitMultipartFileInputStreams(file, shardSize);
long shardCount = inputStreams.size(); //总片数
//封装合并参数
MergeShardArgs mergeShardArgs = new MergeShardArgs((int) shardCount, fileName, md5BucketName, fileType, fileSize);
boolean fileExists = isFileExists(md5BucketName);
boolean bucketExists = minioTemplate.bucketExists(md5BucketName);
//当前文件不存在DB和minio 可以正常分片上传
if (!fileExists && !bucketExists) {
try {
//创建临时桶
minioTemplate.makeBucket(md5BucketName);
uploadJob(shardCount, inputStreams, res, stopStatus, md5BucketName, fileName, userUuid);
//开始合并
Md5FileNameEntry md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
//上传完成清空当前用户数据
res.get(userUuid).clear();
log.info("文件上传成功 {} ", file.getOriginalFilename());
res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_SUCCESS.getCode());
return md5FileNameEntry;
} catch (Exception e) {
log.error("分片合并失败:{}", e.getMessage());
throw new Exception("分片合并失败");
}
}
/*
如果文件存在;
1、存在DB minio == null 上传完成秒传
2、存在minio DB == null
先看临时桶在不在
1、在;断点上传
2、在;没合并
*/
else if (fileExists && !bucketExists) {
//1、存在DB minio == null
md5FileNameMapper.delete(new QueryWrapper<Md5FileNameEntry>()
.eq("md5_file_name", md5BucketName)
);
throw new Exception("请重新上传文件");
} else if (!fileExists) {
//2、存在minio DB == null
// * 1、在;断点上传
// * 2、在;没合并
List<String> objectNames = minioTemplate.listObjectNames(md5BucketName);
Md5FileNameEntry md5FileNameEntry;
//临时桶在; 断点上传
if (objectNames.size() == shardCount) {
//设置百分比
res.get(userUuid).put("uploadPercent", 100);
log.info("uploadPercent:{}", 100);
//设置上传文件大小
res.get(userUuid).put("uploadSize", fileSize);
//没有合并: 合并秒传
md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
log.info("uploadSize:{}", fileSize);
log.info("{} 秒传成功", fileName);
//上传完成清空当前用户数据
res.get(userUuid).clear();
} else {
//断点上传
log.info("开始断点上传>>>>>>");
List<String> containStr = containList(objectNames, shardCount);
log.info("上传过的分片:" + containStr);
CountDownLatch countDownLatch = new CountDownLatch(containStr.size());
try {
log.info("开始断点分片上传:" + fileName);
for (String s : containStr) {
stopStatus = (boolean) res.get("userUuid").get("stopStatus");
if (stopStatus) {
int c = Integer.parseInt(s);
executorService.execute(new BranchThread(inputStreams.get(c - 1), md5BucketName, c, res, countDownLatch, shardCount, stopStatus, userUuid, minioTemplate, sseEmitterService));
} else {
executorService.shutdown();
break;
}
}
} catch (Exception e) {
log.error("断点上传出现异常{}", e.getMessage());
throw new Exception("断点上传出现异常");
} finally {
//关闭线程池
executorService.shutdown();
}
countDownLatch.await();
log.info("所有分片上传完成");
res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getCode());
md5FileNameEntry = mergeShard(mergeShardArgs, userUuid, res);
log.info("文件上传成功:{} ", fileName);
//上传完成清空当前用户数据
res.get(userUuid).clear();
}
res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_MERGE_SUCCESS.getCode());
return md5FileNameEntry;
} else {
//出现异常
log.error("出现异常: {}", ShardFileStatusCode.FOUND.getMessage());
throw new Exception("文件上传出现异常");
}
}
/**
* @description:
* @author wxl
* @date 2022/7/22 22:49
* @param mergeShardArgs 合并文件参数实体
* @param userUuid user id
* @return Md5FileNameEntry
**/
@Transactional
@Override
public Md5FileNameEntry mergeShard(MergeShardArgs mergeShardArgs, String userUuid, ResponseEntry res) {
Integer shardCount = mergeShardArgs.getShardCount();
String fileName = (String) res.get(userUuid).get("fileName");
String md5 = mergeShardArgs.getMd5();
try {
List<String> objectNameList = minioTemplate.listObjectNames(md5);
//查询的服务器的分片和传入的分片不同
if (shardCount != objectNameList.size()) {
// 失败
log.error("服务器的分片{}和传入的分片不同{}", shardCount, objectNameList.size());
throw new Exception("服务器的分片和传入的分片不同");
} else {
// 开始合并请求
String targetBucketName = minioConfig.getBucketName();
//拼接合并之后的文件名称
String objectName = userUuid + "-" + fileName;
//合并
minioTemplate.composeObject(md5, targetBucketName, objectName);
log.info("桶:{} 中的分片文件,已经在桶:{},文件 {} 合并成功", md5, targetBucketName, objectName);
String url = minioTemplate.getPresignedObjectUrl(targetBucketName, objectName);
// 合并成功之后删除对应的临时桶
minioTemplate.removeBucket(md5, true);
log.info("删除桶 {} 成功", md5);
// 存入DB中
Md5FileNameEntry md5FileNameEntry = new Md5FileNameEntry();
md5FileNameEntry.setMd5FileName(md5);
md5FileNameEntry.setUrl(url);
md5FileNameEntry.setCreateTime(new Date());
md5FileNameEntry.setUpdateTime(new Date());
md5FileNameMapper.insert(md5FileNameEntry);
log.info("文件合并成{}并存入DB", md5FileNameEntry);
return md5FileNameEntry;
}
} catch (Exception e) {
// 失败
log.error("合并失败:{}", e.getMessage());
}
return null;
}
/**
* 根据文件大小和文件的md5校验文件是否存在
* 暂时使用Redis实现,后续需要存入数据库
* 实现秒传接口
*
* @param md5 文件的md5
* @return 操作是否成功
*/
@SneakyThrows
@Override
public boolean isFileExists(String md5) {
if (ObjectUtils.isEmpty(md5)) {
log.error("参数md5为空");
throw new Exception("参数md5为空");
}
// 查询
Md5FileNameEntry md5FileNameEntry = md5FileNameMapper.selectOne(new QueryWrapper<Md5FileNameEntry>()
.eq("md5_file_name", md5)
);
/*
文件不存在 false
文件存在 true
*/
return md5FileNameEntry != null;
}
/**
* @description: 小文件上传 0-5MB
* @author wxl
* @date 2022/7/22 22:48
* @param multipartFile file
* @param userUuid user id
* @param voiceTimeSize 声音的长度
* @return Md5FileNameEntry
**/
@Override
public Md5FileNameEntry upload(MultipartFile multipartFile, String userUuid, ResponseEntry res, int voiceTimeSize) throws Exception {
if (multipartFile == null) {
throw new Exception(ShardFileStatusCode.FAILURE.getMessage());
}
if (userUuid == null) {
throw new Exception(ShardFileStatusCode.FILE_IS_NULL.getMessage());
}
log.info(multipartFile.getName());
String fileName = (String) res.get(userUuid).get("fileName");
String targetBucketName = minioConfig.getBucketName();
String targetName;
if (voiceTimeSize <= 0){
targetName = userUuid + "-" + fileName;
}else {
targetName = userUuid + "-" + voiceTimeSize + "-" +fileName;
}
if (isFileExists(targetName)){
log.warn("File is exist already");
throw new Exception("File is exist already");
}
else {
MinioTemplateResult result = minioTemplate.putSimpleObject(multipartFile.getInputStream(), targetBucketName, targetName);
log.info("小文件上传成功");
String url = minioTemplate.getPresignedObjectUrl(targetBucketName, result.getOriginalFileName());
Md5FileNameEntry md5FileNameEntry = new Md5FileNameEntry();
md5FileNameEntry.setMd5FileName(result.getOriginalFileName());
md5FileNameEntry.setUrl(url);
md5FileNameEntry.setCreateTime(new Date());
md5FileNameEntry.setUpdateTime(new Date());
md5FileNameMapper.insert(md5FileNameEntry);
log.info("小文件插入DB");
return md5FileNameEntry;
}
}
/**
* @description: 0-6位取文件名
* @author wxl
* @date 2022/7/22 22:50
* @param oleName oleName
* @return String
**/
@Override
public String rename(String oleName) {
return oleName.substring(0, 6);
}
/**
* @param objNames the object names
* @param shardCount the shard count
* @return List<String>
* @description: 查询上传过的分片
* @author wxl
* @date 2022/7/21 0:00
**/
private List<String> containList(List<String> objNames, long shardCount) {
List<String> containList = new ArrayList<>();
for (int i = 1; i <= shardCount; i++) {
String str = String.valueOf(i);
if (!objNames.contains(str)) {
containList.add(str);
}
}
return containList;
}
/**
* @param shardCount number of shards
* @param inputStreams files stream
* @param res 全局共享消息体
* @param stopStatus 暂停状态
* @param md5BucketName md5 bucket name
* @param fileName file name
* @description:分片任务
* @author wxl
* @date 2022/7/20 23:54
**/
private void uploadJob(long shardCount, List<InputStream> inputStreams, ResponseEntry res, Boolean stopStatus, String md5BucketName, String fileName, String userUuid) throws InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch((int) shardCount);
if (shardCount > 10000) {
throw new RuntimeException("Total parts count should not exceed 10000");
}
log.info("文件分得总片数:" + shardCount);
try {
log.info("开始分片上传:" + fileName);
for (int i = 0; i < shardCount; i++) {
stopStatus = (Boolean) res.get(userUuid).get("stopStatus");
if (stopStatus) {
executorService.execute(new BranchThread(inputStreams.get(i), md5BucketName, i + 1, res, countDownLatch, shardCount, stopStatus, userUuid, minioTemplate, sseEmitterService));
} else {
executorService.shutdown();
break;
}
}
} catch (Exception e) {
log.error("线程上传出现异常:{}", e.getMessage());
} finally {
//关闭线程池
executorService.shutdown();
}
log.info(">>>>>>>>>>等待分片上传");
countDownLatch.await();
log.info(">>>>>>>>>>所有分片上传完成");
res.get(userUuid).put("status", ShardFileStatusCode.ALL_CHUNK_UPLOAD_SUCCESS.getCode());
}
/**
* @author wxl
* @date 2022/7/20 23:57
* @description: 分片上传内部类
**/
private static class BranchThread implements Runnable {
/**
* 文件流
*/
private final InputStream inputStream;
/**
* md5名
*/
private final String md5BucketName;
/**
* 当前片数
*/
private final Integer curIndex;
/**
* 返回给前端的res
*/
private final ResponseEntry res;
/**
* 计数等待线程执行完成
*/
private final CountDownLatch countDownLatch;
/**
* 总片数
*/
private final long shardCount;
/**
* 暂停状态
*/
private final boolean stopStatus;
/**
* 用户id
*/
private final String userUuid;
/**
* template
*/
private final MinioTemplate minioTemplate;
/**
* sse发给前端的服务
*/
private final SseEmitterService sseEmitterService;
public BranchThread(InputStream inputStream, String md5BucketName, Integer curIndex, ResponseEntry res, CountDownLatch countDownLatch, long shardCount, boolean stopStatus, String userUuid, MinioTemplate minioTemplate, SseEmitterService sseEmitterService) {
this.inputStream = inputStream;
this.md5BucketName = md5BucketName;
this.curIndex = curIndex;
this.res = res;
this.countDownLatch = countDownLatch;
this.shardCount = shardCount;
this.stopStatus = stopStatus;
this.userUuid = userUuid;
this.minioTemplate = minioTemplate;
this.sseEmitterService = sseEmitterService;
}
@SneakyThrows
@Override
public void run() {
try {
if (stopStatus) {
Long uploadPercent = ((curIndex * 100) / shardCount);
String curIndexName = String.valueOf(curIndex);
//设置百分比
res.get(userUuid).put("uploadPercent", uploadPercent);
log.info("uploadPercent:{}", uploadPercent);
//设置上传文件大小
res.get(userUuid).put("uploadSize", inputStream.available());
log.info("uploadSize:{}", inputStream.available());
// sseEmitterService.sendResMapToOneClient(userUuid, res);
MinioTemplateResult minioTemplateResult = minioTemplate.putChunkObject(inputStream, md5BucketName, curIndexName);
log.info("分片上传成功: {}", minioTemplateResult);
} else {
executorService.shutdown();
}
} catch (Exception e) {
log.error("线程上传分片异常:{}", e.getMessage());
} finally {
countDownLatch.countDown();
}
}
}
}
3.2、SSE进度消息推送
package com.worldintek.fms.service.impl;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpStatus;
import com.worldintek.common.exception.ApiException;
import com.worldintek.fms.domain.MessageVo;
import com.worldintek.fms.domain.ResponseEntry;
import com.worldintek.fms.service.SseEmitterService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
/**
* @author wxl
* @version 1.0
* @description: SseEmitter发送信息的服务类
* @date 2022/7/8 13:37
*/
@Service
@Slf4j
public class SseEmitterServiceImpl extends SseEmitter implements SseEmitterService {
@Autowired
private ResponseEntry res;
/**
* 容器,保存连接,用于输出返回 ;可使用其他方法实现
*/
private static final Map<String, SseEmitter> sseCache = new ConcurrentHashMap<>();
/**
* 根据客户端id获取SseEmitter对象
*
* @param clientId 客户端ID
*/
@Override
public SseEmitter getSseEmitterByClientId(String clientId) {
return sseCache.get(clientId);
}
/**
* 创建连接
*
* @param clientId 客户端ID
*/
@Override
public SseEmitter createConnect(String clientId) {
if (clientId == null) {
throw new ApiException("用户Id为不能建立连接");
}
// 设置超时时间,0表示不过期。默认30秒,超过时间未完成会抛出异常:AsyncRequestTimeoutException
SseEmitter sseEmitter = new SseEmitter(100000L);
// 是否需要给客户端推送ID
// if (StrUtil.isBlank(clientId)) {
// clientId = IdUtil.simpleUUID();
// res.put(clientId, Map.of("", clientId));
// }
// res.put("clientId", Map.of("clientId", clientId));
log.info("当前clientId----->{}", clientId);
// 注册回调
// 长链接完成后回调接口(即关闭连接时调用)
sseEmitter.onCompletion(completionCallBack(clientId));
// 连接超时回调
sseEmitter.onTimeout(timeoutCallBack(clientId));
// 推送消息异常时,回调方法
sseEmitter.onError(errorCallBack(clientId));
sseCache.put(clientId, sseEmitter);
log.info("创建新的sse连接,当前用户:{} 累计用户:{}", clientId, sseCache.size());
try {
// 注册成功返回用户信息
sseEmitter.send(SseEmitter.event()
.id(clientId)
.data("HttpStatus----->" + HttpStatus.HTTP_CREATED)
);
} catch (IOException e) {
log.error("创建长链接异常,客户端ID:{} 异常信息:{}", clientId, e.getMessage());
}
return sseEmitter;
}
/**
* 发送消息给所有客户端
*
* @param msg 消息内容
*/
@Override
public void sendMessageToAllClient(String msg) {
if (MapUtil.isEmpty(sseCache)) {
return;
}
// 判断发送的消息是否为空
for (Map.Entry<String, SseEmitter> entry : sseCache.entrySet()) {
MessageVo messageVo = new MessageVo();
messageVo.setClientId(entry.getKey());
messageVo.setData(msg);
sendMsgToClientByClientId(entry.getKey(), messageVo, entry.getValue());
}
}
/**
* 给指定客户端发送消息
*
* @param clientId 客户端ID
* @param msg 消息内容
*/
@Override
public void sendMessageToOneClient(String clientId, String msg) {
MessageVo messageVo = new MessageVo(clientId, msg);
sendMsgToClientByClientId(clientId, messageVo, sseCache.get(clientId));
}
/**
* 给指定客户端发送消息
*
* @param clientId 客户端ID
* @param responseEntry 消息内容
*/
@Override
public void sendResMapToOneClient(String clientId, ResponseEntry responseEntry) {
sendResMapToClientByClientId(clientId, responseEntry, sseCache.get(clientId));
}
/**
* @description: 服务器给客户端发送消息
* @author wxl
* @date 2022/7/21 16:00
* @param clientId 客户端ID
**/
@Override
public void sendMessage(String clientId) {
sendResMapToOneClient(clientId, res);
}
/**
* 推送消息到客户端
* 此处做了推送失败后,重试推送机制,可根据自己业务进行修改
*
* @param clientId 客户端ID
* @param responseEntry 推送信息,此处结合具体业务,定义自己的返回值即可
**/
private void sendResMapToClientByClientId(String clientId, ResponseEntry responseEntry, SseEmitter sseEmitter) {
if (sseEmitter == null) {
log.error("推送消息失败:客户端{}未创建长链接,失败消息:{}",
clientId, responseEntry.toString());
return;
}
SseEventBuilder sendData = SseEmitter.event().data("HttpStatus:" + HttpStatus.HTTP_OK, MediaType.APPLICATION_JSON)
.data(responseEntry, MediaType.APPLICATION_JSON);
try {
sseEmitter.send(sendData);
} catch (IOException e) {
// 推送消息失败,记录错误日志,进行重推
log.error("推送消息失败:{},尝试进行重推", responseEntry.toString());
boolean isSuccess = true;
// 推送消息失败后,每隔10s推送一次,推送5次
for (int i = 0; i < 5; i++) {
try {
Thread.sleep(10000);
sseEmitter = sseCache.get(clientId);
if (sseEmitter == null) {
log.error("{}的第{}次消息重推失败,未创建长链接", clientId, i + 1);
continue;
}
sseEmitter.send(sendData);
} catch (Exception ex) {
log.error("{}的第{}次消息重推失败", clientId, i + 1, ex);
continue;
}
log.info("{}的第{}次消息重推成功,{}", clientId, i + 1, responseEntry.toString());
return;
}
}
}
/**
* 推送消息到客户端
* 此处做了推送失败后,重试推送机制,可根据自己业务进行修改
*
* @param clientId 客户端ID
* @param messageVo 推送信息,此处结合具体业务,定义自己的返回值即可
**/
private void sendMsgToClientByClientId(String clientId, MessageVo messageVo, SseEmitter sseEmitter) {
if (sseEmitter == null) {
log.error("推送消息失败:客户端{}未创建长链接,失败消息:{}",
clientId, messageVo.toString());
return;
}
SseEventBuilder sendData = SseEmitter.event().data("HttpStatus:" + HttpStatus.HTTP_OK, MediaType.APPLICATION_JSON)
.data(messageVo, MediaType.APPLICATION_JSON);
try {
sseEmitter.send(sendData);
} catch (IOException e) {
// 推送消息失败,记录错误日志,进行重推
log.error("推送消息失败:{},尝试进行重推", messageVo.toString());
boolean isSuccess = true;
// 推送消息失败后,每隔10s推送一次,推送5次
for (int i = 0; i < 5; i++) {
try {
Thread.sleep(10000);
sseEmitter = sseCache.get(clientId);
if (sseEmitter == null) {
log.error("{}的第{}次消息重推失败,未创建长链接", clientId, i + 1);
continue;
}
sseEmitter.send(sendData);
} catch (Exception ex) {
log.error("{}的第{}次消息重推失败", clientId, i + 1, ex);
continue;
}
log.info("{}的第{}次消息重推成功,{}", clientId, i + 1, messageVo.toString());
return;
}
}
}
/**
* 关闭连接
*
* @param clientId 客户端ID
*/
@Override
public void closeConnect(String clientId) {
SseEmitter sseEmitter = sseCache.get(clientId);
if (sseEmitter != null) {
sseEmitter.complete();
removeUser(clientId);
}
}
/**
* 长链接完成后回调接口(即关闭连接时调用)
*
* @param clientId 客户端ID
**/
private Runnable completionCallBack(String clientId) {
return () -> {
log.info("结束连接:{}", clientId);
removeUser(clientId);
};
}
/**
* 连接超时时调用
*
* @param clientId 客户端ID
**/
private Runnable timeoutCallBack(String clientId) {
return () -> {
log.info("连接超时:{}", clientId);
removeUser(clientId);
};
}
/**
* 推送消息异常时,回调方法
*
* @param clientId 客户端ID
**/
private Consumer<Throwable> errorCallBack(String clientId) {
return throwable -> {
log.error("SseEmitterServiceImpl[errorCallBack]:连接异常,客户端ID:{}", clientId);
// 推送消息失败后,每隔10s推送一次,推送5次
for (int i = 0; i < 5; i++) {
try {
Thread.sleep(10000);
SseEmitter sseEmitter = sseCache.get(clientId);
if (sseEmitter == null) {
log.error("SseEmitterServiceImpl[errorCallBack]:第{}次消息重推失败,未获取到 {} 对应的长链接", i + 1, clientId);
continue;
}
sseEmitter.send("失败后重新推送");
} catch (Exception e) {
e.printStackTrace();
}
}
};
}
/**
* 移除用户连接
*
* @param clientId 客户端ID
**/
private void removeUser(String clientId) {
sseCache.remove(clientId);
log.info("SseEmitterServiceImpl[removeUser]:移除用户:{}", clientId);
}
}