minio简介
- MinIO 是在 Apache License v2.0 下发布的对象存储服务器。 它与 Amazon S3 云存储服务兼容。 它最适合存储非结构化数据,如照片,视频,日志文件,备份和容器/ VM 映像。 对象的大小可以从几 KB 到最大 5TB。
- MinIO 服务器足够轻,可以与应用程序堆栈捆绑在一起,类似于 NodeJS,Redis 和 MySQL。
- 一种高性能的分布式对象存储服务器,用于大型数据基础设施。它是机器学习和其他大数
据工作负载下 Hadoop HDFS 的理想 s3 兼容替代品
为什么要用 Minio
选择它的理由
- Minio 有良好的存储机制
- Minio 有很好纠删码的算法与擦除编码算法
- 拥有RS code 编码数据恢复原理
- 公司做强做大时,数据的拥有重要性,对数据治理与大数据分析做准备。
- 搭建自己的一套文件系统服务,对文件数据进行安全保护。
- 拥有自己的平台,不限于其他方限制。
存储机制
Minio使用纠删码erasure code和校验和checksum来保护数据免受硬件故障和无声数据损坏。 即便丢失一半数量(N/2)的硬盘,仍然可以恢复数据。
直入正题SpringBoot操作minio
引入依赖
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>0.5.3</version>
</dependency>
配置文件
minio:
endpoint:
accessKey:
secretKey:
bucketName: filebucket
配置类
@Configuration
@IntegrationComponentScan
@Slf4j
public class MinioConfig {
@Value("${minio.endpoint}")
private String endpoint;
@Value("${minio.bucketName}")
private String bucketName;
@Value("${minio.accessKey}")
private String accessKey;
@Value("${minio.secretKey}")
private String secretKey;
@Bean
public MinioUtils creatMinioClient() {
return new MinioUtils(endpoint, bucketName, accessKey, secretKey);
}
}
工具类
/**
* @Author: Cyw
* @Date: 2020/12/01/10:02
* @Description: Minio工具类
*/
@Slf4j
public class MinioUtils {
private static MinioClient minioClient;
private static String endpoint;
private static String bucketName;
private static String accessKey;
private static String secretKey;
private static final String SEPARATOR = "/";
private MinioUtils() {
}
public MinioUtils(String endpoint, String bucketName, String accessKey, String secretKey) {
MinioUtils.endpoint = endpoint;
MinioUtils.bucketName = bucketName;
MinioUtils.accessKey = accessKey;
MinioUtils.secretKey = secretKey;
createMinioClient();
}
/**
* 创建minioClient
*/
public void createMinioClient() {
try {
if (null == minioClient) {
log.info("minioClient create start");
minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey)
.build();
createBucket();
log.info("minioClient create end");
}
} catch (Exception e) {
log.error("连接MinIO服务器异常:{}", e);
}
}
/**
* 获取上传文件的基础路径
*
* @return url
*/
public static String getBasisUrl() {
return endpoint + SEPARATOR + bucketName + SEPARATOR;
}
/**
* 初始化Bucket
* 操作存储桶
*
* @throws Exception 异常
*/
private static void createBucket()
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException, RegionConflictException {
if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
}
}
/**
* 验证bucketName是否存在
*
* @return boolean true:存在
*/
public static boolean bucketExists()
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
}
/**
* 创建bucket
*
* @param bucketName bucket名称
*/
public static void createBucket(String bucketName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException, RegionConflictException {
if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
}
}
/**
* 获取存储桶策略
*
* @param bucketName 存储桶名称
* @return json
*/
private JSONObject getBucketPolicy(String bucketName)
throws IOException, InvalidKeyException, InvalidResponseException, BucketPolicyTooLargeException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, InsufficientDataException, ErrorResponseException {
String bucketPolicy = minioClient
.getBucketPolicy(GetBucketPolicyArgs.builder().bucket(bucketName).build());
return JSONObject.parseObject(bucketPolicy);
}
/**
* 获取全部bucket
* <p>
* https://docs.minio.io/cn/java-client-api-reference.html#listBuckets
*/
public static List<Bucket> getAllBuckets()
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.listBuckets();
}
/**
* 根据bucketName获取信息
*
* @param bucketName bucket名称
*/
public static Optional<Bucket> getBucket(String bucketName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.listBuckets().stream().filter(b -> b.name().equals(bucketName)).findFirst();
}
/**
* 根据bucketName删除信息
*
* @param bucketName bucket名称
*/
public static void removeBucket(String bucketName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
}
/**
* 判断文件是否存在
* 操作文件对象
*
* @param bucketName 存储桶
* @param objectName 对象
* @return true:存在
*/
public static boolean doesObjectExist(String bucketName, String objectName) {
boolean exist = true;
try {
minioClient
.statObject(StatObjectArgs.builder().bucket(bucketName).object(objectName).build());
} catch (Exception e) {
exist = false;
}
return exist;
}
/**
* 判断文件夹是否存在
*
* @param bucketName 存储桶
* @param objectName 文件夹名称(去掉/)
* @return true:存在
*/
public static boolean doesFolderExist(String bucketName, String objectName) {
boolean exist = false;
try {
Iterable<Result<Item>> results = minioClient.listObjects(
ListObjectsArgs.builder().bucket(bucketName).prefix(objectName).recursive(false).build());
for (Result<Item> result : results) {
Item item = result.get();
if (item.isDir() && objectName.equals(item.objectName())) {
exist = true;
}
}
} catch (Exception e) {
exist = false;
}
return exist;
}
/**
* 根据文件前置查询文件
*
* @param bucketName bucket名称
* @param prefix 前缀
* @param recursive 是否递归查询
* @return MinioItem 列表
*/
public static List<Item> getAllObjectsByPrefix(String bucketName, String prefix,
boolean recursive)
throws ErrorResponseException, InsufficientDataException, InternalException, InvalidBucketNameException, InvalidKeyException, InvalidResponseException,
IOException, NoSuchAlgorithmException, ServerException, XmlParserException {
List<Item> list = new ArrayList<>();
Iterable<Result<Item>> objectsIterator = minioClient.listObjects(
ListObjectsArgs.builder().bucket(bucketName).prefix(prefix).recursive(recursive).build());
if (objectsIterator != null) {
for (Result<Item> o : objectsIterator) {
Item item = o.get();
list.add(item);
}
}
return list;
}
/**
* 获取文件流
*
* @param bucketName bucket名称
* @param objectName 文件名称
* @return 二进制流
*/
public static InputStream getObject(String bucketName, String objectName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient
.getObject(GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
/**
* 断点下载
*
* @param bucketName bucket名称
* @param objectName 文件名称
* @param offset 起始字节的位置
* @param length 要读取的长度
* @return 流
*/
public InputStream getObject(String bucketName, String objectName, long offset, long length)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.getObject(
GetObjectArgs.builder().bucket(bucketName).object(objectName).offset(offset).length(length)
.build());
}
/**
* 获取路径下文件列表
*
* @param bucketName bucket名称
* @param prefix 文件名称
* @param recursive 是否递归查找,如果是false,就模拟文件夹结构查找
* @return 二进制流
*/
public static Iterable<Result<Item>> listObjects(String bucketName, String prefix,
boolean recursive) {
return minioClient.listObjects(
ListObjectsArgs.builder().bucket(bucketName).prefix(prefix).recursive(recursive).build());
}
/**
* 通过MultipartFile,上传文件
*
* @param bucketName 存储桶
* @param file 文件
* @param objectName 对象名
*/
public static ObjectWriteResponse putObject(String bucketName, MultipartFile file,
String objectName, String contentType)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
InputStream inputStream = file.getInputStream();
return minioClient.putObject(
PutObjectArgs.builder().bucket(bucketName).object(objectName).contentType(contentType)
.stream(
inputStream, inputStream.available(), -1)
.build());
}
/**
* 上传本地文件
*
* @param bucketName 存储桶
* @param objectName 对象名称
* @param fileName 本地文件路径
*/
public static ObjectWriteResponse putObject(String bucketName, String objectName,
String fileName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.uploadObject(
UploadObjectArgs.builder()
.bucket(bucketName).object(objectName).filename(fileName).build());
}
/**
* 通过流上传文件
*
* @param bucketName 存储桶
* @param objectName 文件对象
* @param inputStream 文件流
*/
public static ObjectWriteResponse putObject(String bucketName, String objectName,
InputStream inputStream)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.putObject(
PutObjectArgs.builder().bucket(bucketName).object(objectName).stream(
inputStream, inputStream.available(), -1)
.build());
}
/**
* 创建文件夹或目录
*
* @param bucketName 存储桶
* @param objectName 目录路径
*/
public static ObjectWriteResponse putDirObject(String bucketName, String objectName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.putObject(
PutObjectArgs.builder().bucket(bucketName).object(objectName).stream(
new ByteArrayInputStream(new byte[]{}), 0, -1)
.build());
}
/**
* 获取文件信息, 如果抛出异常则说明文件不存在
*
* @param bucketName bucket名称
* @param objectName 文件名称
*/
public static ObjectStat statObject(String bucketName, String objectName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient
.statObject(StatObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
/**
* 拷贝文件
*
* @param bucketName bucket名称
* @param objectName 文件名称
* @param srcBucketName 目标bucket名称
* @param srcObjectName 目标文件名称
*/
public static ObjectWriteResponse copyObject(String bucketName, String objectName,
String srcBucketName, String srcObjectName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.copyObject(
CopyObjectArgs.builder()
.source(CopySource.builder().bucket(bucketName).object(objectName).build())
.bucket(srcBucketName)
.object(srcObjectName)
.build());
}
/**
* 删除文件
*
* @param bucketName bucket名称
* @param objectName 文件名称
*/
public static void removeObject(String bucketName, String objectName)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, NoSuchAlgorithmException, ServerException, InternalException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
/**
* 批量删除文件
*
* @param bucketName bucket
* @param keys 需要删除的文件列表
* @return
*/
/*public static Iterable<Result<DeleteError>> removeObjects(String bucketName, List<String> keys) {
List<DeleteObject> objects = new LinkedList<>();
keys.forEach(s -> {
objects.add(new DeleteObject(s));
});
return minioClient.removeObjects(
RemoveObjectsArgs.builder().bucket(bucketName).objects(objects).build());
}*/
public static void removeObjects(String bucketName, List<String> keys) {
List<DeleteObject> objects = new LinkedList<>();
keys.forEach(s -> {
objects.add(new DeleteObject(s));
try {
removeObject(bucketName, s);
} catch (Exception e) {
log.error("批量删除失败!error:{}", e);
}
});
}
/**
* 获取文件外链
* 操作Presigned
*
* @param bucketName bucket名称
* @param objectName 文件名称
* @param expires 过期时间 <=7 秒级
* @return url
*/
public static String getPresignedObjectUrl(String bucketName, String objectName,
Integer expires)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, InvalidExpiresRangeException, ServerException, InternalException, NoSuchAlgorithmException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
return minioClient.presignedGetObject(bucketName, objectName, expires);
}
/**
* 给presigned URL设置策略
*
* @param bucketName 存储桶
* @param objectName 对象名
* @param expires 过期策略
* @return map
*/
public static Map<String, String> presignedGetObject(String bucketName, String objectName,
Integer expires)
throws IOException, InvalidKeyException, InvalidResponseException, InsufficientDataException, InvalidExpiresRangeException, ServerException, InternalException, NoSuchAlgorithmException, XmlParserException, InvalidBucketNameException, ErrorResponseException {
PostPolicy policy = new PostPolicy(bucketName, objectName,
ZonedDateTime.now().plusDays(7));
policy.setContentType("image/png");
return minioClient.presignedPostPolicy(policy);
}
/**
* 将URLDecoder编码转成UTF8
*
* @param str
* @return
* @throws UnsupportedEncodingException
*/
public static String getUtf8ByURLDecoder(String str) throws UnsupportedEncodingException {
String url = str.replaceAll("%(?![0-9a-fA-F]{2})", "%25");
return URLDecoder.decode(url, "UTF-8");
}
public static String getUrl(String objectName) throws Exception {
return minioClient.getObjectUrl(bucketName, objectName);
}
}