准备环境
首先在minio配置bucket,bucket名称为:mediafiles,并设置bucket的权限为公开。
配置yaml文件
minio:
endpoint: http://192.168.101.65:9000
accessKey: minioadmin
secretKey: minioadmin
bucket:
files: mediafiles
videofiles: video
编写minio的配置类:
@Configuration
public class MinioConfig {
@Value("${minio.endpoint}")
private String endpoint;
@Value("${minio.accessKey}")
private String accessKey;
@Value("${minio.secretKey}")
private String secretKey;
@Bean
public MinioClient minioClient() {
MinioClient minioClient =
MinioClient.builder()
.endpoint(endpoint)
.credentials(accessKey, secretKey)
.build();
return minioClient;
}
}
分块与合并测试
为了更好的理解文件分块上传的原理,下边用java代码测试文件的分块与合并。
文件分块的流程如下:
1、获取源文件长度
2、根据设定的分块文件的大小计算出块数
3、从源文件读数据依次向每一个块文件写数据。
测试代码如下:
public class BigFileTest {
//测试文件分块方法
@Test
public void testChunk() throws IOException {
File sourceFile = new File("d:/develop/bigfile_test/nacos.mp4");
String chunkPath = "d:/develop/bigfile_test/chunk/";
File chunkFolder = new File(chunkPath);
if (!chunkFolder.exists()) {
chunkFolder.mkdirs();
}
//分块大小
long chunkSize = 1024 * 1024 * 1;
//分块数量
long chunkNum = (long) Math.ceil(sourceFile.length() * 1.0 / chunkSize);
System.out.println("分块总数:"+chunkNum);
//缓冲区大小
byte[] b = new byte[1024];
//使用RandomAccessFile访问文件
RandomAccessFile raf_read = new RandomAccessFile(sourceFile, "r");
//分块
for (int i = 0; i < chunkNum; i++) {
//创建分块文件
File file = new File(chunkPath + i);
if(file.exists()){
file.delete();
}
boolean newFile = file.createNewFile();
if (newFile) {
//向分块文件中写数据
RandomAccessFile raf_write = new RandomAccessFile(file, "rw");
int len = -1;
while ((len = raf_read.read(b)) != -1) {
raf_write.write(b, 0, len);
if (file.length() >= chunkSize) {
break;
}
}
raf_write.close();
System.out.println("完成分块"+i);
}
}
raf_read.close();
}
}
文件合并流程:
1、找到要合并的文件并按文件合并的先后进行排序。
2、创建合并文件
3、依次从合并的文件中读取数据向合并文件写入数
文件合并的测试代码 :
//测试文件合并方法
@Test
public void testMerge() throws IOException {
//块文件目录
File chunkFolder = new File("d:/develop/bigfile_test/chunk/");
//原始文件
File originalFile = new File("d:/develop/bigfile_test/nacos.mp4");
//合并文件
File mergeFile = new File("d:/develop/bigfile_test/nacos01.mp4");
if (mergeFile.exists()) {
mergeFile.delete();
}
//创建新的合并文件
mergeFile.createNewFile();
//用于写文件
RandomAccessFile raf_write = new RandomAccessFile(mergeFile, "rw");
//指针指向文件顶端
raf_write.seek(0);
//缓冲区
byte[] b = new byte[1024];
//分块列表
File[] fileArray = chunkFolder.listFiles();
// 转成集合,便于排序
List<File> fileList = Arrays.asList(fileArray);
// 从小到大排序
Collections.sort(fileList, new Comparator<File>() {
@Override
public int compare(File o1, File o2) {
return Integer.parseInt(o1.getName()) - Integer.parseInt(o2.getName());
}
});
//合并文件
for (File chunkFile : fileList) {
RandomAccessFile raf_read = new RandomAccessFile(chunkFile, "rw");
int len = -1;
while ((len = raf_read.read(b)) != -1) {
raf_write.write(b, 0, len);
}
raf_read.close();
}
raf_write.close();
//校验文件
try (
FileInputStream fileInputStream = new FileInputStream(originalFile);
FileInputStream mergeFileStream = new FileInputStream(mergeFile);
) {
//取出原始文件的md5
String originalMd5 = DigestUtils.md5Hex(fileInputStream);
//取出合并文件的md5进行比较
String mergeFileMd5 = DigestUtils.md5Hex(mergeFileStream);
if (originalMd5.equals(mergeFileMd5)) {
System.out.println("合并文件成功");
} else {
System.out.println("合并文件失败");
}
}
}
minio合并文件测试
将分块文件上传至minio
//将分块文件上传至minio
@Test
public void uploadChunk(){
String chunkFolderPath = "D:\\develop\\upload\\chunk\\";
File chunkFolder = new File(chunkFolderPath);
//分块文件
File[] files = chunkFolder.listFiles();
//将分块文件上传至minio
for (int i = 0; i < files.length; i++) {
try {
UploadObjectArgs uploadObjectArgs = UploadObjectArgs.builder().bucket("testbucket").object("chunk/" + i).filename(files[i].getAbsolutePath()).build();
minioClient.uploadObject(uploadObjectArgs);
System.out.println("上传分块成功"+i);
} catch (Exception e) {
e.printStackTrace();
}
}
}
通过minio的合并文件
//合并文件,要求分块文件最小5M
@Test
public void test_merge() throws Exception {
List<ComposeSource> sources = Stream.iterate(0, i -> ++i)
.limit(6)
.map(i -> ComposeSource.builder()
.bucket("testbucket")
.object("chunk/".concat(Integer.toString(i)))
.build())
.collect(Collectors.toList());
ComposeObjectArgs composeObjectArgs = ComposeObjectArgs.builder().bucket("testbucket").object("merge01.mp4").sources(sources).build();
minioClient.composeObject(composeObjectArgs);
}
//清除分块文件
@Test
public void test_removeObjects(){
//合并分块完成将分块文件清除
List<DeleteObject> deleteObjects = Stream.iterate(0, i -> ++i)
.limit(6)
.map(i -> new DeleteObject("chunk/".concat(Integer.toString(i))))
.collect(Collectors.toList());
RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket("testbucket").objects(deleteObjects).build();
Iterable<Result<DeleteError>> results = minioClient.removeObjects(removeObjectsArgs);
results.forEach(r->{
DeleteError deleteError = null;
try {
deleteError = r.get();
} catch (Exception e) {
e.printStackTrace();
}
});
}
使用minio合并文件报错:java.lang.IllegalArgumentException: source testbucket/chunk/0: size 1048576 must be greater than 5242880 minio合并文件默认分块最小5M,我们将分块改为5M再次测试。
上传分块开发
检查文件和分块
接口完成进行接口实现,首先实现检查文件方法和检查分块方法。
/**
* @description 检查文件是否存在
* @param fileMd5 文件的md5
* @return com.xuecheng.base.model.RestResponse<java.lang.Boolean> false不存在,true存在
* @author Mr.M
* @date 2022/9/13 15:38
*/
public RestResponse<Boolean> checkFile(String fileMd5);
/**
* @description 检查分块是否存在
* @param fileMd5 文件的md5
* @param chunkIndex 分块序号
* @return com.xuecheng.base.model.RestResponse<java.lang.Boolean> false不存在,true存在
* @author Mr.M
* @date 2022/9/13 15:39
*/
public RestResponse<Boolean> checkChunk(String fileMd5, int chunkIndex);
service接口实现方法:
@Override
public RestResponse<Boolean> checkFile(String fileMd5) {
//查询文件信息
MediaFiles mediaFiles = mediaFilesMapper.selectById(fileMd5);
if (mediaFiles != null) {
//桶
String bucket = mediaFiles.getBucket();
//存储目录
String filePath = mediaFiles.getFilePath();
//文件流
InputStream stream = null;
try {
stream = minioClient.getObject(
GetObjectArgs.builder()
.bucket(bucket)
.object(filePath)
.build());
if (stream != null) {
//文件已存在
return RestResponse.success(true);
}
} catch (Exception e) {
}
}
//文件不存在
return RestResponse.success(false);
}
@Override
public RestResponse<Boolean> checkChunk(String fileMd5, int chunkIndex) {
//得到分块文件目录
String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
//得到分块文件的路径
String chunkFilePath = chunkFileFolderPath + chunkIndex;
//文件流
InputStream fileInputStream = null;
try {
fileInputStream = minioClient.getObject(
GetObjectArgs.builder()
.bucket(bucket_videoFiles)
.object(chunkFilePath)
.build());
if (fileInputStream != null) {
//分块已存在
return RestResponse.success(true);
}
} catch (Exception e) {
}
//分块未存在
return RestResponse.success(false);
}
//得到分块文件的目录
private String getChunkFileFolderPath(String fileMd5) {
return fileMd5.substring(0, 1) + "/" + fileMd5.substring(1, 2) + "/" + fileMd5 + "/" + "chunk" + "/";
}
上传分块
定义service接口
/**
* @description 上传分块
* @param fileMd5 文件md5
* @param chunk 分块序号
* @param localChunkFilePath 分块文件本地路径
* @return com.xuecheng.base.model.RestResponse
* @author Mr.M
* @date 2022/9/13 15:50
*/
public RestResponse uploadChunk(String fileMd5,int chunk,String localChunkFilePath);
接口实现:
@Override
public RestResponse uploadChunk(String fileMd5, int chunk, String localChunkFilePath) {
//得到分块文件的目录路径
String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
//得到分块文件的路径
String chunkFilePath = chunkFileFolderPath + chunk;
//mimeType
String mimeType = getMimeType(null);
//将文件存储至minIO
boolean b = addMediaFilesToMinIO(localChunkFilePath, mimeType, bucket_videoFiles, chunkFilePath);
if (!b) {
log.debug("上传分块文件失败:{}", chunkFilePath);
return RestResponse.validfail(false, "上传分块失败");
}
log.debug("上传分块文件成功:{}",chunkFilePath);
return RestResponse.success(true);
}
上传分块测试
@ApiOperation(value = "文件上传前检查文件")
@PostMapping("/upload/checkfile")
public RestResponse<Boolean> checkfile(
@RequestParam("fileMd5") String fileMd5
) throws Exception {
return mediaFileService.checkFile(fileMd5);
}
@ApiOperation(value = "分块文件上传前的检测")
@PostMapping("/upload/checkchunk")
public RestResponse<Boolean> checkchunk(@RequestParam("fileMd5") String fileMd5,
@RequestParam("chunk") int chunk) throws Exception {
return mediaFileService.checkChunk(fileMd5,chunk);
}
@ApiOperation(value = "上传分块文件")
@PostMapping("/upload/uploadchunk")
public RestResponse uploadchunk(@RequestParam("file") MultipartFile file, @RequestParam("fileMd5") String fileMd5,
@RequestParam("chunk") int chunk) throws Exception {
//创建临时文件
File tempFile = File.createTempFile("minio", "temp");
//上传的文件拷贝到临时文件
file.transferTo(tempFile);
//文件路径
String absolutePath = tempFile.getAbsolutePath();
return mediaFileService.uploadChunk(fileMd5,chunk,absolutePath);
}
启动前端工程,进入上传视频界面进行前后端联调测试。
前端对文件分块的大小为5MB,SpringBoot web默认上传文件的大小限制为1MB,这里需要在media-api工程修改配置如下:
spring:
servlet:
multipart:
max-file-size: 50MB
max-request-size: 50MB
max-file-size:单个文件的大小限制
Max-request-size: 单次请求的大小限制
合并分块开发
定义service接口:
/**
* @description 合并分块
* @param companyId 机构id
* @param fileMd5 文件md5
* @param chunkTotal 分块总和
* @param uploadFileParamsDto 文件信息
* @return com.xuecheng.base.model.RestResponse
* @author Mr.M
* @date 2022/9/13 15:56
*/
public RestResponse mergechunks(Long companyId,String fileMd5,int chunkTotal,UploadFileParamsDto uploadFileParamsDto);
service实现:
@Override
public RestResponse mergechunks(Long companyId, String fileMd5, int chunkTotal, UploadFileParamsDto uploadFileParamsDto) {
//=====获取分块文件路径=====
String chunkFileFolderPath = getChunkFileFolderPath(fileMd5);
//组成将分块文件路径组成 List<ComposeSource>
List<ComposeSource> sourceObjectList = Stream.iterate(0, i -> ++i)
.limit(chunkTotal)
.map(i -> ComposeSource.builder()
.bucket(bucket_videoFiles)
.object(chunkFileFolderPath.concat(Integer.toString(i)))
.build())
.collect(Collectors.toList());
//=====合并=====
//文件名称
String fileName = uploadFileParamsDto.getFilename();
//文件扩展名
String extName = fileName.substring(fileName.lastIndexOf("."));
//合并文件路径
String mergeFilePath = getFilePathByMd5(fileMd5, extName);
try {
//合并文件
ObjectWriteResponse response = minioClient.composeObject(
ComposeObjectArgs.builder()
.bucket(bucket_videoFiles)
.object(mergeFilePath)
.sources(sourceObjectList)
.build());
log.debug("合并文件成功:{}",mergeFilePath);
} catch (Exception e) {
log.debug("合并文件失败,fileMd5:{},异常:{}",fileMd5,e.getMessage(),e);
return RestResponse.validfail(false, "合并文件失败。");
}
// ====验证md5====
//下载合并后的文件
File minioFile = downloadFileFromMinIO(bucket_videoFiles,mergeFilePath);
if(minioFile == null){
log.debug("下载合并后文件失败,mergeFilePath:{}",mergeFilePath);
return RestResponse.validfail(false, "下载合并后文件失败。");
}
try (InputStream newFileInputStream = new FileInputStream(minioFile)) {
//minio上文件的md5值
String md5Hex = DigestUtils.md5Hex(newFileInputStream);
//比较md5值,不一致则说明文件不完整
if(!fileMd5.equals(md5Hex)){
return RestResponse.validfail(false, "文件合并校验失败,最终上传失败。");
}
//文件大小
uploadFileParamsDto.setFileSize(minioFile.length());
}catch (Exception e){
log.debug("校验文件失败,fileMd5:{},异常:{}",fileMd5,e.getMessage(),e);
return RestResponse.validfail(false, "文件合并校验失败,最终上传失败。");
}finally {
if(minioFile!=null){
minioFile.delete();
}
}
//文件入库
currentProxy.addMediaFilesToDb(companyId,fileMd5,uploadFileParamsDto,bucket_videoFiles,mergeFilePath);
//=====清除分块文件=====
clearChunkFiles(chunkFileFolderPath,chunkTotal);
return RestResponse.success(true);
}
/**
* 从minio下载文件
* @param bucket 桶
* @param objectName 对象名称
* @return 下载后的文件
*/
public File downloadFileFromMinIO(String bucket,String objectName){
//临时文件
File minioFile = null;
FileOutputStream outputStream = null;
try{
InputStream stream = minioClient.getObject(GetObjectArgs.builder()
.bucket(bucket)
.object(objectName)
.build());
//创建临时文件
minioFile=File.createTempFile("minio", ".merge");
outputStream = new FileOutputStream(minioFile);
IOUtils.copy(stream,outputStream);
return minioFile;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(outputStream!=null){
try {
outputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 得到合并后的文件的地址
* @param fileMd5 文件id即md5值
* @param fileExt 文件扩展名
* @return
*/
private String getFilePathByMd5(String fileMd5,String fileExt){
return fileMd5.substring(0,1) + "/" + fileMd5.substring(1,2) + "/" + fileMd5 + "/" +fileMd5 +fileExt;
}
/**
* 清除分块文件
* @param chunkFileFolderPath 分块文件路径
* @param chunkTotal 分块文件总数
*/
private void clearChunkFiles(String chunkFileFolderPath,int chunkTotal){
try {
List<DeleteObject> deleteObjects = Stream.iterate(0, i -> ++i)
.limit(chunkTotal)
.map(i -> new DeleteObject(chunkFileFolderPath.concat(Integer.toString(i))))
.collect(Collectors.toList());
RemoveObjectsArgs removeObjectsArgs = RemoveObjectsArgs.builder().bucket("video").objects(deleteObjects).build();
Iterable<Result<DeleteError>> results = minioClient.removeObjects(removeObjectsArgs);
results.forEach(r->{
DeleteError deleteError = null;
try {
deleteError = r.get();
} catch (Exception e) {
e.printStackTrace();
log.error("清楚分块文件失败,objectname:{}",deleteError.objectName(),e);
}
});
} catch (Exception e) {
e.printStackTrace();
log.error("清楚分块文件失败,chunkFileFolderPath:{}",chunkFileFolderPath,e);
}
}
接口层
@ApiOperation(value = "合并文件")
@PostMapping("/upload/mergechunks")
public RestResponse mergechunks(@RequestParam("fileMd5") String fileMd5,
@RequestParam("fileName") String fileName,
@RequestParam("chunkTotal") int chunkTotal) throws Exception {
Long companyId = 1232141425L;
UploadFileParamsDto uploadFileParamsDto = new UploadFileParamsDto();
uploadFileParamsDto.setFileType("001002");
uploadFileParamsDto.setTags("课程视频");
uploadFileParamsDto.setRemark("");
uploadFileParamsDto.setFilename(fileName);
return mediaFileService.mergechunks(companyId,fileMd5,chunkTotal,uploadFileParamsDto);
}
合并分块测试
下边进行前后端联调:
1、上传一个视频测试合并分块的执行逻辑
进入service方法逐行跟踪。
2、断点续传测试
上传一部分后,停止刷新浏览器再重新上传,通过浏览器日志发现已经上传过的分块不再重新上传