minio 工具类_minio 8.5-CSDN博客

本文链接：https://blog.csdn.net/weixin_43987869/article/details/134839492

Minio 工具类

前言

使用8.5.7的jar包

 <dependency>
            <groupId>io.minio</groupId>
            <artifactId>minio</artifactId>
            <version>8.5.7</version>
        </dependency>

详细代码

实现了线程池大文件分块上传以及分块下载

文件分块上传：先将文件进行分块，多线程上传到临时桶。然后全部上传完成之后，再调用composeObject（）方法进行合并，再删除临时文件、临时桶。
分块下载：调用getObject（）方法指定偏移量和大小然后多线程获取流，然后使用RandomAccessFile加锁根据偏移量同步写入到指定的文件

import io.minio.*;
import io.minio.errors.*;
import io.minio.http.Method;
import io.minio.messages.DeleteError;
import io.minio.messages.DeleteObject;
import io.minio.messages.Item;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.*;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.*;

/**
 * @ClassName:
 * @Description:
 * @Author: 
 * @Date: 2023/12/2 18:59
 * @Version: 1.0
 */
@Component
@Slf4j
public class MinioUtils {
    @Resource
    private MinioClient minioClient;

    /**
     *  #5M 单位围为M
     */
    @Value("${minio.chunk}")
    private int chunk;

    /**
     *  #分块合并大小
     */
    @Value("${minio.chunkSize}")
    private int chunkSize;

    private static final Object lock = new Object();

    /**
     * 判断bucket是否存在
     *
     * @return
     */
    public boolean bucketExists(String bucketName) {
        boolean exists = false;
        try {
            if (StringUtils.isNotBlank(bucketName))
                exists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
        } catch (Exception e) {
            log.error("判断bucket是否存在 异常e", e.getMessage());
            e.printStackTrace();
        }
        return exists;
    }

    /**
     * 创建桶
     */
    public boolean createBucket(String bucketName) {
        try {
            if (!bucketExists(bucketName)) {
                minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
                return true;
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return false;
    }


    /**
     * 删除桶
     */
    public void removeBucket(String bucketName) throws Exception {
        if (bucketExists(bucketName))
            minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
    }

    /**
     * 获取minio中,某个bucket中所有的文件名
     */
    public Map<String, String> getFileList(String bucketName) {
        Map<String, String> fileUrlData = new HashMap<>();
        if (bucketExists(bucketName)) {
            Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName)
                    .recursive(true).build());
            String fileName = null;
            Item item;
            String url;
            for (Result<Item> result : results) {
                try {
                    item = result.get();
                    fileName = item.objectName();
                    url = getUrl(fileName, item.objectName());
                    fileUrlData.put(fileName, url);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
        return fileUrlData;
    }

    /**
     * 上传
     */
    public boolean uploadFile(String fileName,String bucketName, InputStream stream, Long fileSize, String type) {
        try {
            minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).stream(stream, fileSize, -1)
                    .contentType(type).build());
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    }

    /**
     * 分块上传
     *
     * @return
     */
    public boolean uploadChunkedFile(String filePath, String fileName, InputStream stream, String bucketName) {
        try {
            //作临时桶
            String filePrefix = fileName.substring(0, fileName.lastIndexOf("."));
            if (!bucketExists(filePrefix))
                minioClient.makeBucket(MakeBucketArgs.builder().bucket(filePrefix).build());
            if (!bucketExists(bucketName))
                minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());

            //创建后续需要分块的集合
            List<ComposeSource> sourceObjectList = new ArrayList<ComposeSource>();

            FileInputStream fis = new FileInputStream(filePath);
            long fileSize = fis.getChannel().size();
            int chunksCount = (int) Math.ceil((double) fileSize / (chunk * 1024 * 1024));

            ThreadPoolExecutor executor = new ThreadPoolExecutor(
                    10, // 核心线程数
                    20, // 最大线程数
                    60L, // 线程空闲超时时间（单位：秒）
                    TimeUnit.SECONDS, // 线程空闲超时时间单位
                    new ArrayBlockingQueue<>(chunksCount), // 任务队列大小
                    Executors.defaultThreadFactory(), // 线程工厂
                    new ThreadPoolExecutor.AbortPolicy() // 拒绝策略
            );

            upload(filePrefix, fileName, chunksCount, (int) chunk * 1024 * 1024, fis, executor);
            List<DeleteObject> objects = new LinkedList<DeleteObject>();
            for (int i = 0; i < chunksCount; i++) {
                objects.add(new DeleteObject(fileName + i));
                sourceObjectList.add(ComposeSource.builder().bucket(filePrefix).object(fileName + i).build());
            }

            merge(filePrefix, fileName, bucketName, sourceObjectList, objects);

            //删除最后一次合并的文件
            deletes(filePrefix, objects);
            //删除临时桶
            removeBucket(filePrefix);
            return true;
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
    }

    /**
     * 分块上传
     *
     * @param fileName
     * @param filePrefix
     * @param chunksCount
     * @param chunkSize
     * @param fis
     * @param executor
     * @throws IOException
     */
    private void upload(String filePrefix, String fileName, int chunksCount, int chunkSize, FileInputStream fis, ThreadPoolExecutor executor) throws IOException {
        log.info("开始准备上传---------------------------------------------");
        // 依次上传每个分块。
        for (int i = 0; i < chunksCount; i++) {
            int index = i;
            byte[] buffer = new byte[chunkSize];
            int bytesRead = fis.read(buffer);

            if (bytesRead == -1) {
                break;
            }
            executor.execute(new Runnable() {
                @Override
                public void run() {
                    // 上传每个分块。
                    try {
                        //log.info("--------------------------------------");
                        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(buffer);
                        minioClient.putObject(
                                PutObjectArgs.builder()
                                        .bucket(filePrefix)
                                        .object(fileName + index)
                                        .stream(byteArrayInputStream, bytesRead, -1)
                                        .build());
                        byteArrayInputStream.close();
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });
        }
        // 等待所有的分片完成
        // shutdown方法：通知各个任务（Runnable）的运行结束
        executor.shutdown();
        while (!executor.isTerminated()) {
            try {
                // 指定的时间内所有的任务都结束的时候，返回true，反之返回false，返回false还有执行完的任务
                executor.awaitTermination(3, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                log.error(e.getMessage());
            }
        }
        fis.close();
        log.info("上传完成---------------------------------------------");
    }

    /**
     * 合并
     *
     * @param fileName
     * @param bucketName
     * @param filePrefix       临时桶
     * @param sourceObjectList
     * @throws ErrorResponseException
     * @throws InsufficientDataException
     * @throws InternalException
     * @throws InvalidKeyException
     * @throws InvalidResponseException
     * @throws IOException
     * @throws NoSuchAlgorithmException
     * @throws ServerException
     * @throws XmlParserException
     */
    private void merge(String filePrefix, String fileName, String bucketName, List<ComposeSource> sourceObjectList, List<DeleteObject> objects)
            throws ErrorResponseException, InsufficientDataException, InternalException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException {
        ThreadPoolExecutor executor;
        int sourceSize = (int) Math.ceil((double) sourceObjectList.size() / chunkSize);
        if (sourceSize > 1) {
            executor = new ThreadPoolExecutor(
                    10, // 核心线程数
                    20, // 最大线程数
                    60L, // 线程空闲超时时间（单位：秒）
                    TimeUnit.SECONDS, // 线程空闲超时时间单位
                    new ArrayBlockingQueue<>(sourceSize), // 任务队列大小
                    Executors.defaultThreadFactory(), // 线程工厂
                    new ThreadPoolExecutor.AbortPolicy() // 拒绝策略
            );

            for (int i = 0; i < sourceSize; i++) {
                int index = i;
                int start = i * chunkSize;
                int end = Math.min(start + chunkSize, sourceObjectList.size());
                log.info("start: {}, ebd: {}", start, end);
                List<ComposeSource> composeSources = sourceObjectList.subList(start, end);
                executor.submit(() -> {

                    // 调用composeObject方法合并分片
                    try {
                        List<ComposeSource> subSources = composeSources;
                        minioClient.composeObject(ComposeObjectArgs.builder()
                                .bucket(filePrefix)
                                .object(fileName + "-" + index)
                                .sources(subSources)
                                .build());
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                });
            }
            // 等待所有的分片完成
            // shutdown方法：通知各个任务（Runnable）的运行结束
            executor.shutdown();
            while (!executor.isTerminated()) {
                try {
                    // 指定的时间内所有的任务都结束的时候，返回true，反之返回false，返回false还有执行完的任务
                    executor.awaitTermination(3, TimeUnit.SECONDS);
                } catch (InterruptedException e) {
                    log.error(e.getMessage());
                }
            }

            sourceObjectList = new ArrayList<>();
            for (int i = 0; i < sourceSize; i++) {
                sourceObjectList.add(ComposeSource.builder().bucket(filePrefix).object(fileName + "-" + i).build());
                objects.add(new DeleteObject(fileName + "-" + i));
            }
        }

        minioClient.composeObject(ComposeObjectArgs.builder()
                .bucket(bucketName)
                .object(fileName)
                .sources(sourceObjectList)
                .build());
        log.info("合并完成---------------------------------------------");
    }

    /**
     * 分块下载
     */
    public boolean downloadChunkedFile(String filePath, String fileName, String bucketName) {
        if (!bucketExists(bucketName)) return false;

        try {
            StatObjectResponse statObjectResponse = minioClient.statObject(StatObjectArgs.builder().bucket(bucketName).object(fileName).build());

            int totalParts = (int) Math.ceil((double) statObjectResponse.size() / (chunk * 1024 * 1024));

            ThreadPoolExecutor executor = new ThreadPoolExecutor(
                    10, // 核心线程数
                    20, // 最大线程数
                    60L, // 线程空闲超时时间（单位：秒）
                    TimeUnit.SECONDS, // 线程空闲超时时间单位
                    new ArrayBlockingQueue<>(totalParts + 1), // 任务队列大小
                    Executors.defaultThreadFactory(), // 线程工厂
                    new ThreadPoolExecutor.AbortPolicy() // 拒绝策略
            );

            for (int i = 0; i < totalParts; i++) {
                long offset = i;
                executor.execute(new Runnable() {
                    @Override
                    public void run() {
                        synchronized (statObjectResponse) {
                            InputStream inputStream = null;
                            long index = offset;
                            long length = chunk * 1024 * 1024;
                            index = index * length;
                            if (index + length > statObjectResponse.size()) {
                                length = statObjectResponse.size() - index;
                                log.info("最后一次：offset:{} , index:{},  let: {} , cout:{} ", offset, index, length, statObjectResponse.size());
                            }
                            log.info("offset:{} , index:{},  let: {} , cout:{} ", offset, index, length, statObjectResponse.size());
                            // 下载
                            try {
                                inputStream = minioClient.getObject(GetObjectArgs
                                        .builder()
                                        .bucket(bucketName)
                                        .object(fileName)
                                        .length(length)
                                        .offset(index)
                                        .build());

                                weiteToFile(inputStream, filePath, index);

                            } catch (Exception e) {
                                throw new RuntimeException(e);
                            } finally {
                                if (null != inputStream) {
                                    try {
                                        inputStream.close();
                                    } catch (IOException e) {
                                        throw new RuntimeException(e);
                                    }
                                }
                            }
                        }
                    }
                });
            }
            executor.shutdown();
            while (!executor.isTerminated()) {
                try {
                    // 指定的时间内所有的任务都结束的时候，返回true，反之返回false，返回false还有执行完的任务
                    executor.awaitTermination(3, TimeUnit.SECONDS);
                } catch (InterruptedException e) {
                    log.error(e.getMessage());
                }
            }
            return true;
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }

    }

    /**
     * 保存到文件
     * @param inputStream
     * @param filePath
     * @param offset
     */
    private void weiteToFile(InputStream inputStream, String filePath, long offset) {
        byte[] buffer = new byte[1024];
        int bytesRead;
        try (RandomAccessFile file = new RandomAccessFile(filePath, "rw")){
            synchronized (lock) {
                file.seek(offset);
                while ((bytesRead = inputStream.read(buffer)) != -1) {
                    file.write(buffer, 0, bytesRead);
                }
                file.getFD().sync();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 直接下载
     * @param bucketName
     * @param objectName
     * @param fileName
     * @return
     * @throws Exception
     */
    public boolean download(String bucketName, String objectName, String fileName) {

        try {
            minioClient.downloadObject(DownloadObjectArgs
                    .builder()
                    .bucket(bucketName)
                    .object(objectName)
                    .filename(fileName)
                    .build());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return false;
    }

    /**
     * 获取文件流
     *
     * @param bucketName bucket名称
     * @param objectName 文件名称
     * @return 二进制流
     */
    public InputStream getObject(String bucketName, String objectName)
            throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, ErrorResponseException {
        return minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
    }


    /**
     * 判断文件夹是否存在
     *
     * @return
     */
    public Boolean folderExists(String bucketName, String prefix) throws Exception {
        Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).prefix(
                prefix).recursive(false).build());
        for (Result<Item> result : results) {
            Item item = result.get();
            if (item.isDir()) {
                return true;
            }
        }
        return false;
    }

    /**
     * 创建文件夹
     *
     * @param bucketName 桶名称
     * @param path       路径
     */
    public void createFolder(String bucketName, String path) throws Exception {
        minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(path)
                .stream(new ByteArrayInputStream(new byte[]{}), 0, -1).build());
    }

    /**
     * 获取文件在minio在服务器上的外链
     */
    public String getUrl(String objectName, String bucketName) throws Exception {
        return minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder().method(Method.GET).bucket(
                bucketName).object(objectName).expiry(1, TimeUnit.DAYS).build());
    }

    /**
     * 删除文件
     *
     * @param bucketName
     * @param fileName
     */
    public boolean delete(String bucketName, String fileName) {
        if (bucketExists(bucketName)) {

            if (statObject(bucketName, fileName)) {
                try {
                    minioClient.removeObject(
                            RemoveObjectArgs.builder().bucket(bucketName).object(fileName).build());
                    return true;
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        }
        return false;
    }

    /**
     * 批量删除文件
     *
     * @param bucketName
     * @param list
     */
    public boolean deletes(String bucketName, List<DeleteObject> list) {
        if (bucketExists(bucketName)) {
            try {
                Iterable<Result<DeleteError>> results =
                        minioClient.removeObjects(
                                RemoveObjectsArgs.builder().bucket(bucketName).objects(list).build());
                for (Result<DeleteError> result : results) {
                    DeleteError error = result.get();
                    System.out.println(
                            "Error in deleting object " + error.objectName() + "; " + error.message());
                }
                return true;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        return false;
    }

    /**
     * 判断文件是否存在
     *
     * @param bucketName
     * @param fileName
     */
    public boolean statObject(String bucketName, String fileName) {
        if (bucketExists(bucketName)) {
            try {
                minioClient.statObject(
                        StatObjectArgs.builder().bucket(bucketName).object(fileName).build());
                return true;
            } catch (Exception e) {
                if (e.getMessage().equals("Object does not exist")) {
                    return false;
                }
            }
        }
        return false;
    }

}

多线程写入同一个文件时，需要确保线程之间的操作是同步的，以避免竞态条件和文件损坏。以下是一些可能导致问题的常见原因和解决方法：
文件指针位置问题： 多个线程同时写入时，需要确保每个线程写入的位置不会相互干扰。
解决方法：使用文件锁或其他同步机制确保每个线程写入的位置是独占的。在Java中，可以使用RandomAccessFile类，并通过synchronized关键字或其他同步工具来确保线程安全。
缓冲区刷新问题： 如果每个线程都有自己的缓冲区，可能会导致数据被乱序写入文件，从而破坏文件内容。
解决方法： 使用共享的缓冲区或在每次写入后强制刷新缓冲区。可以使用flush()方法来刷新缓冲区。
文件覆盖问题：多个线程可能会尝试同时覆盖相同的文件内容。
解决方法：使用文件锁或其他同步机制来确保每个线程在写入文件之前检查文件的状态，以防止覆盖。