Spring Cloud：微服务之构建支持多存储方式的分布式文件存储服务

概述

在微务系统架构中，分布式文件系统（Distributed File System,DFS）是必不可少的中间件。分布式文件系统是面对互联网的需求而产生，互联网时代对海量数据如何存储？靠简单的增加硬盘的个数已经满足不了我们的要求，因为硬盘传输速度有限但是数据在急剧增长，另外我们还要要做好数据备份、数据安全等。通过分布式文件系统我们可以将相同的文件同时存储到网络上多台服务器上，从而实现提供文件的访问效率和提高文件的可用性。

目前各大主流云服务商都提供对象存储服务（OSS）,也存在许多成熟的开源本地化部署分布式文件系统解决方案（如FastDFS，MinIO等），使我们可以很方便的在业务系统是使用。

今天我们讨论的重点并不是如何搭建一个分布式文件系统，而是如何在微服务系统中搭建一个同时兼容多种DFS的文件存储微服务。

当前搭建的file微服务支持四种存储方式，本地存储、FastDFS存储、MinIO存储和阿里云OSS存储，可以通过配置动态切换。

项目目录

本地存储

1、配置参数

  file:
    type: LOCAL
    local:
      domain: http://127.0.0.1:6500
      path: D:/uploadPath
	  bucketName: /upload

2、继承WebMvcConfigurer，重写addResourceHandlers，在registry里面配置访问路径和映射到的服务器本地路径。

@Configuration
@RequiredArgsConstructor
public class ResourceConfigurer implements WebMvcConfigurer {

    private final FileProperties fileProperties;

    @Override
    public void addResourceHandlers(ResourceHandlerRegistry registry) {
        //映射上传静态文件到文件目录
        registry.addResourceHandler(fileProperties.getLocal().getBucketName() + "/**")
                .addResourceLocations("file:" + fileProperties.getLocal().getPath() + File.separator);
    }
}

3、实现本地文件上传逻辑

/**
 * @description: 本地文件存储
 */
public class LocalFileStorageClient implements FileStorageClient {

    /**
     * 默认大小 50M
     */
    public static final long DEFAULT_MAX_SIZE = 50 * 1024 * 1024;

    /**
     * 默认的文件名最大长度 100
     */
    public static final int DEFAULT_FILE_NAME_LENGTH = 100;

    /**
     * 默认可上传文件格式
     */
    public static final String[] DEFAULT_ALLOWED_EXTENSION = {
            // 图片
            "bmp", "gif", "jpg", "jpeg", "png",
            // word excel powerpoint
            "doc", "docx", "xls", "xlsx", "ppt", "pptx", "html", "htm", "txt",
            // 压缩文件
            "rar", "zip", "gz", "bz2",
            // 视频格式
            "mp4", "avi", "rmvb",
            // pdf
            "pdf"};


    @Override
    public String uploadFile(MultipartFile file, FileProperties fileProperties) throws Exception {

        String fileName = file.getOriginalFilename();
        int fileNameLength = fileName.length();
        if (fileNameLength > DEFAULT_FILE_NAME_LENGTH) {
            throw new FileException("upload.filename.exceed.length: " + DEFAULT_FILE_NAME_LENGTH);
        }

        //校验文件
        assertAllowed(file, DEFAULT_ALLOWED_EXTENSION);

        //生成文件名
        String extension = FilenameUtils.getExtension(file.getOriginalFilename());
        fileName = DateUtils.dateTimeNow("yyyyMMdd") + "/" + UUID.randomUUID().toString().replace("-", "") + "." + extension;

        //保持本地
        File desc = getAbsoluteFile(fileProperties.getLocal().getPath(), fileName);
        file.transferTo(desc);

        //生成文件路径
        String url = fileProperties.getLocal().getDomain() + fileProperties.getLocal().getBucketName() + "/" + fileName;

        return url;
    }

    /**
     * 文件大小校验
     *
     * @param file 上传的文件
     * @throws FileException 文件校验异常
     */
    private void assertAllowed(MultipartFile file, String[] allowedExtension) throws FileException {
        long size = file.getSize();
        if (DEFAULT_MAX_SIZE != -1 && size > DEFAULT_MAX_SIZE) {
            throw new FileException("upload.exceed.maxSize: " + DEFAULT_MAX_SIZE / 1024 / 1024);
        }

        String fileName = file.getOriginalFilename();
        String extension = FilenameUtils.getExtension(file.getOriginalFilename());

        if (allowedExtension != null && !isAllowedExtension(extension, allowedExtension)) {
            throw new FileException("filename : [" + fileName + "], extension : [" + extension + "], allowed extension : [" + Arrays.toString(allowedExtension) + "]");
        }
    }

    /**
     * 判断MIME类型是否是允许的MIME类型
     *
     * @param extension        上传文件类型
     * @param allowedExtension 允许上传文件类型
     * @return true/false
     */
    private boolean isAllowedExtension(String extension, String[] allowedExtension) {
        for (String str : allowedExtension) {
            if (str.equalsIgnoreCase(extension)) {
                return true;
            }
        }
        return false;
    }

    /**
     * 获取绝对文件
     *
     * @param path:
     * @param fileName:
     * @return java.io.File
     */
    private File getAbsoluteFile(String path, String fileName) {
        File desc = new File(path + File.separator + fileName);

        if (!desc.exists()) {
            if (!desc.getParentFile().exists()) {
                desc.getParentFile().mkdirs();
            }
        }
        return desc.isAbsolute() ? desc : desc.getAbsoluteFile();
    }
}

4、自动配置

    @Configuration(proxyBeanMethods = false)
    @ConditionalOnProperty(name = {"spring2go.file.type"}, havingValue = FileStorageType.LOCAL)
    protected static class LocalStorageConfiguration {
        protected LocalStorageConfiguration() {
        }

        @Bean
        public LocalFileStorageClient localFileStorageClient() {
            return new LocalFileStorageClient();
        }
    }

5、编写文件上传接口

@RestController
@RequiredArgsConstructor
@Slf4j
@RequestMapping("/file")
public class FileController {
    private final FileStorageClient client;
    private final FileProperties fileProperties;

    /**
     * 上传文件
     * 文件名采用uuid,避免原始文件名中带"-"符号导致下载的时候解析出现异常
     *
     * @param file 资源
     * @return R
     */
    @PostMapping("/upload")
    public R upload(@RequestParam("file") MultipartFile file, HttpServletRequest request) {
        try {
            String url = client.uploadFile(file, fileProperties);
            return R.ok(url);
        } catch (Exception e) {
            log.error("上传文件失败", e);
            return R.failed("上传文件失败：" + e.getMessage());
        }
    }
}

FastDFS存储

1、添加依赖

        <dependency>
            <groupId>com.github.tobato</groupId>
            <artifactId>fastdfs-client</artifactId>
            <version>1.27.2</version>
        </dependency>

2、配置参数

  file:
    type: FASTDFS
    fastdfs:
      domain: http://127.0.0.1:6500
	 
# TODO 暂时使用fastdfs-client的配置项，后续优化整合配置
# FastDFS配置
fdfs:
  domain: http://127.0.0.1
  soTimeout: 3000
  connectTimeout: 2000
  trackerList: 127.0.0.1:22122

3、 自动配置

    @Configuration(proxyBeanMethods = false)
    @ConditionalOnProperty(name = {"spring2go.file.type"}, havingValue = FileStorageType.FASTDFS)
    protected static class FastdfsStorageConfiguration {
        protected FastdfsStorageConfiguration() {
        }

        @Bean
        public FastDfsFileStorageClient fastDfsFileStorageClient(FastFileStorageClient storageClient) {
            return new FastDfsFileStorageClient(storageClient);
        }
    }

4、实现上传逻辑

@RequiredArgsConstructor
public class FastDfsFileStorageClient implements FileStorageClient {

    private final FastFileStorageClient storageClient;

    @Override
    public String uploadFile(MultipartFile file, FileProperties fileProperties) throws Exception {
        StorePath storePath = storageClient.uploadFile(file.getInputStream(), file.getSize(),
                FilenameUtils.getExtension(file.getOriginalFilename()), null);
        String url = fileProperties.getFastdfs().getDomain() + "/" + storePath.getFullPath();
        return url;
    }
}

5、复用上传接口逻辑

MinIO存储

1、添加引用

        <dependency>
            <groupId>io.minio</groupId>
            <artifactId>minio</artifactId>
            <version>8.2.1</version>
		</dependency>

2、配置参数

  file:
    type: MINIO
    minio:
      url: http://127.0.0.1:9000
      accessKey: ""
      secretKey: ""
      bucketName: upload

3、自动配置

    @Configuration(proxyBeanMethods = false)
    @ConditionalOnProperty(name = {"spring2go.file.type"}, havingValue = FileStorageType.MINIO)
    protected static class MinioStorageConfiguration {
        protected MinioStorageConfiguration() {
        }

        @Bean
        @ConditionalOnMissingBean(MinioClient.class)
        public MinioClient minioClient(FileProperties fileProperties) {
            FileProperties.MinioFileProperties minioFileProperties = fileProperties.getMinio();
            return MinioClient.builder().endpoint(minioFileProperties.getUrl()).credentials(minioFileProperties.getAccessKey(), minioFileProperties.getSecretKey()).build();
        }

        @Bean
        public MinioFileStorageClient minioFileStorageClient(MinioClient minioClient) {
            return new MinioFileStorageClient(minioClient);
        }
    }

4、实现上传逻辑

@RequiredArgsConstructor
public class MinioFileStorageClient implements FileStorageClient {

    private final MinioClient minioClient;

    @Override
    public String uploadFile(MultipartFile file, FileProperties fileProperties) throws Exception {
        //生成文件名
        String extension = FilenameUtils.getExtension(file.getOriginalFilename());
        String fileName = DateUtils.dateTimeNow("yyyyMMdd") + "/" + UUID.randomUUID().toString().replace("-", "") + "." + extension;

        PutObjectArgs args = PutObjectArgs.builder()
                .bucket(fileProperties.getMinio().getBucketName())
                .object(fileName)
                .stream(file.getInputStream(), file.getSize(), -1)
                .contentType(file.getContentType())
                .build();
        minioClient.putObject(args);
        String url = fileProperties.getMinio().getUrl() + "/" + fileProperties.getMinio().getBucketName() + "/" + fileName;

        return url;
    }
}

5、复用上传接口逻辑

阿里云OSS存储

1、添加引用

        <dependency>
            <groupId>com.aliyun.oss</groupId>
            <artifactId>aliyun-sdk-oss</artifactId>
            <version>3.12.0</version>
        </dependency>

2、配置参数

  file:
    type: ALIOSS
    alioss:
      endpoint: http://127.0.0.1:9000
      accessKey: ""
      secretKey: ""
      bucketName: upload
      staticDomain: http://127.0.0.1:9000

3、自动配置

    @Configuration(proxyBeanMethods = false)
    @ConditionalOnProperty(name = {"spring2go.file.type"}, havingValue = FileStorageType.ALIOSS)
    protected static class AliossStorageConfiguration {
        protected AliossStorageConfiguration() {
        }

        @Bean
        @ConditionalOnMissingBean(OSSClient.class)
        public OSSClient alioss(FileProperties fileProperties) {
            FileProperties.AliossFileProperties aliossFileProperties = fileProperties.getAlioss();
            OSSClient ossClient = new OSSClient(aliossFileProperties.getEndpoint(),
                    new DefaultCredentialProvider(aliossFileProperties.getAccessKey(), aliossFileProperties.getSecretKey()),
                    new ClientConfiguration());
            return ossClient;
        }

        @Bean
        public AliossFileStorageClient aliossFileStorageClient(OSSClient ossClient) {
            return new AliossFileStorageClient(ossClient);
        }
    }

4、上传逻辑

@RequiredArgsConstructor
public class AliossFileStorageClient implements FileStorageClient {

    private final OSSClient ossClient;

    @Override
    public String uploadFile(MultipartFile file, FileProperties fileProperties) throws IOException {

        //生成文件名
        String extension = FilenameUtils.getExtension(file.getOriginalFilename());
        String fileName = DateUtils.dateTimeNow("yyyyMMdd") + "/" + UUID.randomUUID().toString().replace("-", "") + "." + extension;

        String newBucket = fileProperties.getAlioss().getBucketName();
        //判断桶是否存在,不存在则创建桶
        if (!ossClient.doesBucketExist(newBucket)) {
            ossClient.createBucket(newBucket);
        }
        String url = fileProperties.getAlioss().getStaticDomain() + "/" + fileName;
        ossClient.putObject(newBucket, url, file.getInputStream());
        return url;
    }
}

5、复用上传接口逻辑

（完）