大数据之minio简单使用

简介

Minio 是一个基于Apache License v2.0开源协议的对象存储服务。类似阿里OSS。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。
Minio使用纠删码erasure code和校验和checksum。 即便丢失一半数量（N/2）的硬盘，仍然可以恢复数据,数据可读,丢失（N/2）-1数据可写。
Minio是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL。

架构

安装

方式一：离线安装

创建minio目录
mkdir /opt/software/minio && cd /opt/software/minio

下载安装包

wget http://dl.minio.org.cn/server/minio/release/linux-amd64/minio
wget https://dl.min.io/client/mc/release/linux-amd64/mc

赋执行权限

chmod +x minio mc

创建启动脚本，编辑run.sh文件
vim run.sh

#!/bin/bash
export MINIO_ACCESS_KEY=minio
export MINIO_SECRET_KEY=Wxl#20210910
/opt/software/minio/minio server --console-address '0.0.0.0:9999' \
http://192.168.100.100/opt/software/minio/data1 \
http://192.168.100.101/opt/software/minio/data1 http://192.168.100.101/opt/software/minio/data2 \
http://192.168.100.102/opt/software/minio/data1 http://192.168.100.102/opt/software/minio/data2

赋权执行

chmod 777 /opt/software/minio/run.sh

分发

scp -r /opt/software/minio hadoop101:/opt/software/
scp -r /opt/software/minio hadoop102:/opt/software/

创建启动服务，创建minio.service启动脚本

vim /usr/lib/systemd/system/minio.service

[Unit]
Description=Minio service
Documentation=https://docs.minio.io/
Wants=network-online.target
After=network-online.target

[Service]
WorkingDirectory=/opt/software/minio/
ExecStart=/opt/software/minio/run.sh

# Let systemd restart this service on-failure
Restart=on-failure
RestartSec=5

# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=65536

# Disable timeout logic and wait until process is stopped
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target

重新加载服务的配置文件

systemctl daemon-reload

启动minio服务

systemctl start minio

访问
http://192.168.100.100:9999
http://192.168.100.101:9999
http://192.168.100.102:9999

查看minio状态

systemctl status minio

关闭minio服务

systemctl stop minio

方法二：docker方式

也可通过docker-compose方式一键部署：传送门

docker run --name minio \
-p 9000:9000 \
-p 9001:9001 \
-d --restart=always \
-e "MINIO_ACCESS_KEY=minio" \
-e "MINIO_SECRET_KEY=minio123456" \
-v /etc/localtime:/etc/localtime:ro \
-v /opt/dockerData/minio/data1:/data1 \
-v /opt/dockerData/minio/data2:/data2 \
-v /opt/dockerData/minio/data3:/data3 \
-v /opt/dockerData/minio/data4:/data4 \
-v /opt/dockerData/minio/config:/root/.minio \
minio/minio server /data1 /data2 /data3 /data4 --console-address ":9001"

minio clinet(mc)操作minio

JAVA操作

JAVA SDK: https://docs.min.io/docs/java-client-api-reference.html

<!--minio maven-->
<dependency>
    <groupId>io.minio</groupId>
    <artifactId>minio</artifactId>
    <version>8.0.3</version>
</dependency>

MinIoClientConfig.java

@Component
public class MinIoClientConfig {
    @Value("${minio.endpoint}")
    private String endpoint;
    @Value("${minio.port}")
    private int port;
    @Value("${minio.accessKey}")
    private String accessKey;
    @Value("${minio.secretKey}")
    private String secretKey;

    /**
     * 注入minio 客户端
     *
     * @return
     */
    @Bean
    public MinioClient minioClient() {


        return MinioClient.builder()
                .endpoint(endpoint,port,false)
                .credentials(accessKey, secretKey)
                .build();
    }
}

MinioUtil.java

@Component
public class MinioUtil {

    @Autowired
    private MinioClient minioClient;
    @Autowired
    private ResultInfo resultInfo;


    /**
     * 查看存储bucket是否存在
     *
     * @param bucketName 存储bucket名称
     * @return
     */
    public Boolean bucketExists(String bucketName) {
        Boolean found;
        try {
            BucketExistsArgs bucketExistsArgs
                    = BucketExistsArgs.builder().bucket(bucketName).build();
            found = minioClient.bucketExists(bucketExistsArgs);


        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return found;
    }

    /**
     * 创建存储bucket
     *
     * @param bucketName 存储bucket名称
     * @return Boolean
     */
    public Boolean makeBucket(String bucketName) {
        try {


            minioClient.makeBucket(
                    MakeBucketArgs.builder()
                            .bucket(bucketName)
                            .build());

        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return true;
    }


    /**
     * 删除存储bucket
     *
     * @param bucketName 存储bucket名称
     * @return Boolean
     */
    public Boolean removeBucket(String bucketName) {
        try {
            minioClient.removeBucket(RemoveBucketArgs.builder()
                    .bucket(bucketName)
                    .build());
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return true;
    }

    /**
     * 查询bucket列表
     *
     * @return List<String>
     */


    public List<String> listBuckets() {
        List<String> list = new ArrayList<>();
        try {
            ListBucketsArgs.builder().build();
            List<Bucket> buckets = minioClient.listBuckets(ListBucketsArgs.builder().build());
            for (Bucket bucket : buckets) {
                list.add(bucket.name());
            }
            return list;

        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }

    }

    /**
     * 按名称查找bucket
     *
     * @param bucketName 存储bucket名称
     * @return Bucket
     */

    public Bucket findByBucketName(String bucketName) {
        try {

            ListBucketsArgs listBucketsArgs = ListBucketsArgs.builder().build();

            List<Bucket> buckets = minioClient.listBuckets(listBucketsArgs);
            for (Bucket bucket : buckets) {
                if (bucketName.equals(bucket.name())) {
                    return bucket;
                }
            }


        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
        return null;

    }

    /**
     * 文件上传
     *
     * @param bucketName 存储bucket
     * @param file       文件
     * @return Boolean
     */
    public String upload(String bucketName, MultipartFile file) {
//        String fileName =file.getOriginalFilename();
//        yyyymmddhhmiss+0+4位随机数
        String fileName = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + "0"
                + Math.round(Math.random() * 10000)
                + file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf("."));

        try {
            PutObjectArgs putObjectArgs
                    = PutObjectArgs.builder().bucket(bucketName)
                    .object(fileName)
                    .stream(file.getInputStream(), file.getSize(), -1)
                    .contentType(file.getContentType())
                    .build();

            //文件名称相同会覆盖
            minioClient.putObject(putObjectArgs);


        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
        return fileName;
    }

    public String uploadBatch(String bucketName, MultipartFile[] file) {
        List<String> list = new ArrayList<>();
        for (MultipartFile multipartFile : file) {
            String upload = upload(bucketName, multipartFile);
            list.add(upload);
        }
        return list.toString();
    }

    /**
     * 文件下载
     *
     * @param bucketName 存储bucket名称
     * @param fileName   文件名称
     * @param res        response
     * @return Boolean
     */
    public void download(String bucketName, String fileName, HttpServletResponse res) {
        GetObjectArgs objectArgs = GetObjectArgs.builder().bucket(bucketName)
                .object(fileName).build();
        try (GetObjectResponse response = minioClient.getObject(objectArgs)) {
            byte[] buf = new byte[1024];
            int len;
            try (FastByteArrayOutputStream os = new FastByteArrayOutputStream()) {
                while ((len = response.read(buf)) != -1) {
                    os.write(buf, 0, len);
                }
                os.flush();
                byte[] bytes = os.toByteArray();
                res.setCharacterEncoding("utf-8");
                //设置强制下载不打开
                res.setContentType("application/force-download");
                res.addHeader("Content-Disposition", "attachment;fileName=" + fileName);
                try (ServletOutputStream stream = res.getOutputStream()) {
                    stream.write(bytes);
                    stream.flush();
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void download1(String bucketName, String fileName, HttpServletResponse httpResponse) {
        try {
            //1.下载文件
            InputStream object = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(fileName).build());

            byte buf[] = new byte[1024];
            int length = 0;

            httpResponse.reset();
            httpResponse.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(fileName, "UTF-8"));
            httpResponse.setContentType("application/octet-stream");
            httpResponse.setCharacterEncoding("utf-8");
            OutputStream outputStream = httpResponse.getOutputStream();
            while ((length = object.read(buf)) > 0) {
                outputStream.write(buf, 0, length);
            }
            outputStream.close();
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    /**
     * 查看文件对象
     *
     * @param bucketName 存储bucket名称
     * @return 存储bucket内文件对象信息
     */
    public List<Map<String, Object>> listObjects(String bucketName) {


        List<Map<String, Object>> objectItems = new ArrayList<>();
        try {
            Iterable<Result<Item>> results = minioClient.listObjects(
                    ListObjectsArgs.builder().bucket(bucketName).build());
            for (Result<Item> result : results) {
                Item item = result.get();
                Map<String, Object> map = new HashMap<>();
                map.put("name", item.objectName());
                map.put("size", item.size() + "kb");
                map.put("isDir", item.isDir());
                map.put("lastModified", item.lastModified());
                objectItems.add(map);
            }
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
        return objectItems;
    }

    /**
     * 查看文件对象url
     *
     * @param bucketName 存储bucket名称
     * @return 存储bucket内文件对象信息
     */
    public String findUrl(String bucketName, String fileName) throws Exception {
        try {
//    查看文件地址
            GetPresignedObjectUrlArgs build
                    = new GetPresignedObjectUrlArgs().builder().bucket(bucketName)
                    .object(fileName)
                    .method(Method.GET)
                    .build();
            String url = minioClient.getPresignedObjectUrl(build);
            return url;
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }


    }

    /**
     * 批量删除文件对象
     *
     * @param bucketName 存储bucket名称
     * @param objects    对象名称集合
     */
    public Iterable<Result<DeleteError>> removeObjects(String bucketName, List<String> objects) {
        List<DeleteObject> dos = objects.stream().map(e -> new DeleteObject(e)).collect(Collectors.toList());
        Iterable<Result<DeleteError>> results = minioClient.removeObjects(RemoveObjectsArgs.builder().bucket(bucketName).objects(dos).build());
        return results;
    }