Java基于Minio大文件zip上传

概要

本项目所用技术：1、minio 2、Rabbit MQ

大文件ZIP下载思路与分析

1、本地原始视频 -> Minio -> Web端下载 -> zip打包 -> zip压缩包
2、本地原始视频 -> Minio-> zip打包 -> Web端下载 -> zip压缩包

思路1：在Web端下载时，才去minio下载视频，并进行zip打包，因为视频是大文件，让用户等太久是不可取的
思路2：将zip打包的动作前置，即在用户下载时，zip压缩包已经准备好了，用户友好

ZIP压缩

1、触发视频打包（可选项，根据具体业务适用）
使用Rabbit MQ监听视频打包的请求

rabbitTemplate.convertAndSend("zipUndoQueue", video_.getId());

2、zip打包
使用线程池ThreadPoolTaskExecutor执行打包

@RabbitListener(queues = "zipUndoQueue")
    @RabbitHandler
    public void compressVideos(String videoIdStr) {
        log.info("发起压缩任务,视频id:{}", videoIdStr);
        Video video = videoMapper.selectOne(Wrappers.lambdaQuery(Video.class)
                .select(Video::getId, Video::getVideoName, Video::getUploadFileId)
                .eq(Video::getType, Video.FILE_MP4)
                .eq(Video::getCompressStatus, Video.COMPRESS_UNCOMPRESS)
                .eq(Video::getId, Long.parseLong(videoIdStr))
                .last("for update"));
        if (video != null) {
            executor.execute(() -> {
                try {
                    videoToZip.compressVideo(video);
                } catch (Exception e) {
                    log.error("压缩流程异常,视频信息:{}", video);
                }
            });
        }
    }
/**
     * 创建压缩包
     *
     * @param zipFilePath
     * @param mp4UploadFile
     * @param k4UploadFiles
     * @param videoName
     * @return
     */
    @SneakyThrows
    private void compressFiles(String zipFilePath, UploadFile mp4UploadFile, List<UploadFile> k4UploadFiles, String videoName) {
        log.debug("创建压缩包,视频名称:{}", videoName);
        try (FileOutputStream fos = new FileOutputStream(zipFilePath);
             BufferedOutputStream bos = new BufferedOutputStream(fos);
             // 创建ZipArchiveOutputStream对象并设置参数
             ZipArchiveOutputStream zos = new ZipArchiveOutputStream(bos)) {

            zos.setLevel(Deflater.DEFAULT_COMPRESSION);
            zos.setMethod(ZipArchiveOutputStream.DEFLATED);
            zos.setUseZip64(Zip64Mode.Always);
            // 将mp4添加到压缩文件里
            log.debug("加入mp4文件,视频名称:{},文件内容:{}", videoName, objectMapper.writeValueAsString(mp4UploadFile));
            addToZip(zos, mp4UploadFile, "");
            if (k4UploadFiles != null) {
                // 将高清视频添加到压缩文件里
                for (UploadFile k4UploadFile : k4UploadFiles) {
                    log.debug("加入高清视频文件,视频名称:{},文件内容:{}", videoName, objectMapper.writeValueAsString(k4UploadFile));
                    addToZip(zos, k4UploadFile, k4Prefix);
                }
            }
        } catch (Exception e) {
            log.error("压缩文件异常,视频名称:{},{}", videoName, e);
            throw e;
        }
    }

    /**
     * 压缩包加入文件
     *
     * @param zos
     * @param uploadFile
     * @param prefix
     */
    @SneakyThrows
    private void addToZip(ZipArchiveOutputStream zos, UploadFile uploadFile, String prefix) {
        byte[] buffer = new byte[1024 * 16];
        int bytesRead;
        // 创建新的ZIP条目
        ZipArchiveEntry entry = new ZipArchiveEntry(prefix + File.separator + uploadFile.getFilename());
        zos.putArchiveEntry(entry);
        // 读取源文件内容并写入ZIP文件
        InputStream is = new BufferedInputStream(pearlMinioClient.getObject(
                GetObjectArgs.builder().bucket(uploadFile.getBucketName()).object(uploadFile.getObjectName()).build()));

        while ((bytesRead = is.read(buffer)) != -1) {
            zos.write(buffer, 0, bytesRead);
        }

        zos.closeArchiveEntry();
        IOUtils.closeQuietly(is);
    }

3、上传zip压缩包

    /**
     * 上传zip
     *
     * @param videoCompressRecord
     * @throws Exception
     */
    private void uploadZip(VideoCompressRecord videoCompressRecord) throws Exception {
        
        try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(videoCompressRecord.getZipFilePath()))) {
            pearlMinioClient.putObject(
                    PutObjectArgs.builder()
                            .bucket(zipBucketName)
                            .object(zipObjectName).stream(bis, uploadFile.getTotalSize(), -1)
                            .build());
            videoCompressRecord.setStatus(VideoCompressRecord.COMPRESS_UPLOAD_ZIP); // 压缩包上传成功
            videoCompressRecord.setUpdateTime(new Date());
            videoCompressRecordMapper.updateById(videoCompressRecord);
        } catch (Exception e) {
            log.error("zip上传异常,视频名称:{},压缩文件名称:{},{}", videoCompressRecord.getVideoName(), zipName, e);
            videoCompressRecord.setUpdateTime(new Date());
            videoCompressRecord.setFailed(true);
            videoCompressRecord.setErrorMsg("视频压缩包上传失败:" + Throwables.getStackTraceAsString(e));
            videoCompressRecordMapper.updateById(videoCompressRecord);
            return;
        }
        rabbitTemplate.convertAndSend("zipFinishQueue", videoCompressRecord.getVideoId());
        executor.execute(() -> this.deleteOriginal4K(videoCompressRecord.getVideoId(), uploadFile.getId()));
    }

4、删除临时文件

executor.execute(() -> this.deleteOriginal4K(videoCompressRecord.getVideoId(), uploadFile.getId()));

/**
     * 删除原始4k视频文件
     *
     * @param mp4VideoId
     * @param zipUploadFileId
     */
    private void deleteOriginal4K(Long mp4VideoId, Long zipUploadFileId) {
        List<Video> k4Videos = videoMapper.selectList(Wrappers.lambdaQuery(Video.class)
                .select(Video::getId, Video::getUploadFileId)
                .eq(Video::getType, Video.FILE_4K)
                .eq(Video::getUploadId, mp4VideoId));
        if (!k4Videos.isEmpty()) {
            List<UploadFile> k4UploadFiles = uploadFileMapper.selectList(Wrappers.lambdaQuery(UploadFile.class)
                    .select(UploadFile::getBucketName, UploadFile::getObjectName)
                    .in(UploadFile::getId, k4Videos.stream().map(Video::getUploadFileId).collect(Collectors.toList())));
            for (UploadFile k4UploadFile : k4UploadFiles) {
                try {
                    pearlMinioClient.removeObject(RemoveObjectArgs.builder()
                            .bucket(k4UploadFile.getBucketName())
                            .object(k4UploadFile.getObjectName())
                            .build());
                } catch (Exception e) {
                    log.error("删除原始高清视频文件异常,异常信息:{}", Throwables.getStackTraceAsString(e));
                }
            }
            videoMapper.update(null, Wrappers.lambdaUpdate(Video.class)
                    .eq(Video::getZipUploadFileId, zipUploadFileId)
                    .in(Video::getId, k4Videos.stream().map(Video::getId).collect(Collectors.toList())));
        }
    }

	if (isDeleteZip) {
	                zipFile.delete();
	                if (parentFile != null && parentFile.list().length == 0) {
	                    parentFile.delete();
	                }
	            }
	

5、重新上传zip压缩包
解决因为网络原因导致上传失败的情况

@Scheduled(cron = "0 * * * * ?")
    public void scheduledUploadZip() {
        List<VideoCompressRecord> unUploadVideoZips = videoCompressRecordMapper.selectList(Wrappers.lambdaQuery(VideoCompressRecord.class)
                .eq(VideoCompressRecord::isFailed, true)
                .in(VideoCompressRecord::getStatus, List.of(VideoCompressRecord.COMPRESS_CREATE_ZIP, VideoCompressRecord.COMPRESS_MD5_ZIP)));
        if (unUploadVideoZips != null) {
            for (VideoCompressRecord unUploadVideoZip : unUploadVideoZips) {
                log.info("重新上传zip,打包记录id:{}", unUploadVideoZip.getId());
                unUploadVideoZip.setFailed(false);
                unUploadVideoZip.setErrorMsg("重试上传中");
                unUploadVideoZip.setUpdateTime(new Date());
                videoCompressRecordMapper.updateById(unUploadVideoZip);
                executor.execute(() -> {
                    try {
                        videoToZip.retryUploadZipFile(unUploadVideoZip);
                    } catch (Exception e) {
                        log.error("重新上传zip异常,打包记录id:{}", unUploadVideoZip.getId());
                    }
                });
            }
        }
    }

/**
     * 重新上传zip
     *
     * @param videoCompressRecord
     */
    public void retryUploadZipFile(VideoCompressRecord videoCompressRecord) {
        if (videoCompressRecord.getStatus() != VideoCompressRecord.COMPRESS_CREATE_ZIP && videoCompressRecord.getStatus() != VideoCompressRecord.COMPRESS_MD5_ZIP) {
            return;
        }
        try {
            this.uploadZip(videoCompressRecord);
            if (isDeleteZip) {
                File zipFile = new File(videoCompressRecord.getZipFilePath());
                zipFile.delete();
                if (zipFile.getParentFile().list().length == 0) {
                    zipFile.getParentFile().delete();
                }
            }
        } catch (Exception e) {
            log.error("压缩文件异常,视频名称:{},异常:{}", videoCompressRecord.getVideoName(), e);
            videoCompressRecord.setUpdateTime(new Date());
            videoCompressRecord.setFailed(true);
            videoCompressRecord.setErrorMsg(Throwables.getStackTraceAsString(e));
            videoCompressRecordMapper.updateById(videoCompressRecord);
        }
    }

总结

1、为什么选择MQ，而不是定时任务去实现zip压缩？
首先，不管是定时任务还是MQ，都是异步处理，实现了应用解耦，到底如何选择，需要根据项目的需求而定。
本项目的业务需求：

1. 在视频审核通过后进行zip压缩（事件驱动，逐条处理）
2. 视频数量将会很大（海量数据）
3. 视频审核通过后需要尽快完成zip压缩（实时响应）

用定时任务的话需要每隔一段时间做一次全表扫描，消耗性能
视频压缩的动作仅在视频审核通过后，而MQ就是事件驱动的