SpringBoot搭建分布式文件存储FastDFS

简介：

FastDFS是一个开源的轻量级分布式文件系统，对文件进行管理，功能：文件同步、文件存储、文件访问等，解决了大容量存储和负载均衡的问题。

FastDFS充分考虑了冗余备份、负载均衡、线性扩容机制等注重高可用、高性能指标等，使用FastDFS很容易搭建一套高性能的文件服务器集群提供文件上传、下载服务等。

工作原理：

ContOS7安装FastDFS：

1.使用的网络模式是-net=host，XXX.XXX.XXX.XXX是宿主机的IP
2.group1是组名，既Storage的组
3.如果要增加新的Storage服务器，再次运行改命令(注意更换组名称)

#拉取镜像
docker pull myfastdfs/fastdfs
#运行tracker
docker run -d --name tracker --net=host myfastdfs/fastdfs sh tracker.sh
#运行storage
docker run -d --name storage --net=host -e TRACKER_IP=XXX.XXX.XXX.XXX:22122 -e GROUP_NAME=group1 myfastdfs/fastdfs sh storage.sh

构建Spring Boot微服务：

父工程pom

<parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.6.RELEASE</version>
    </parent>
    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>
    <dependencies>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.51</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
        </dependency>
    </dependencies>
    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-dependencies</artifactId>
                <version>Greenwich.SR1</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>   

my-file-store父工程my-eureka子工程pom文件

<dependencies>
   <dependency>
     <groupId>org.springframework.cloud</groupId>
     <artifactId>spring-cloud-starter-netflix-eureka-server</artifactId>
   </dependency>
</dependencies>

my-file-store父工程my-file子工程pom文件

<dependencies>
   <dependency>
      <groupId>com.filestore</groupId>
      <artifactId>my-file-store</artifactId>
      <version>1.0-SNAPSHOT</version>
   </dependency>
</dependencies>

my-file-store父工程my-file子子pom文件

<dependencies>
   <dependency>
      <groupId>net.oschina.zcx7878</groupId>
      <artifactId>fastdfs-client-java</artifactId>
      <version>1.27.0.0</version>
   </dependency>
   <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-netflix-eureka-client</artifactId>
   </dependency>
</dependencies>

EurekaApplication启动类

package com.filestore;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer;

/**
 * @author : musir
 */
@SpringBootApplication
@EnableEurekaServer
public class EurekaApplication {
    public static void main(String[] args) {
        SpringApplication.run(EurekaApplication.class, args);
    }
}

application.yml配置文件

server:
  port: 7001
eureka:
  instance:
    hostname: 127.0.0.1
  client:
    register-with-eureka: false #是否将自己注册到eureka中
    fetch-registry: false #是否从eureka中获取信息
    service-url:
      defaultZone: http://${eureka.instance.hostname}:${server.port}/eureka/ #动态IP获取
spring:
  application:
    name: eureka

controller文件FileUploadController

package com.changgou.controller;

import com.changgou.file.FastDFSFile;
import com.changgou.util.FastDFSUtil;
import entity.Result;
import entity.StatusCode;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

/**
 * @author ：musir
 */
@RestController
@RequestMapping(value = "/upload")
@CrossOrigin
public class FileUploadController {

    @PostMapping
    public Result upload(@RequestParam(value = "file") MultipartFile multipartFile) throws Exception {
        FastDFSFile fastDFSFile = new FastDFSFile(
                multipartFile.getOriginalFilename(),
                multipartFile.getBytes(),
                StringUtils.getFilenameExtension(multipartFile.getOriginalFilename())
        );
        String[] uploads = FastDFSUtil.upload(fastDFSFile);
        // 拼接访问地址url=http://192.168.xxx.xxx:8080/group1/M00/00/00/XXX.jpg
        // String url = "http://192.168.xxx.xxx:8080/"+uploads[0]+"/"+uploads[1];
        String url = FastDFSUtil.getTrackerInfo()+"/"+uploads[0]+"/"+uploads[1];
        return new Result(true, StatusCode.OK, "上传成功", url);
    }
}

file文件FastDFSFile

package com.changgou.file;

import java.io.Serializable;

/**
 * @author ：musir
 */
public class FastDFSFile implements Serializable {

    // 文件名字
    private String name;
    // 文件内容
    private byte[] content;
    // 文件扩展名
    private String ext;
    // 文件MD5摘要值
    private String md5;
    // 文件创建者
    private String author;

    /**
     * 构造函数
     * @param name
     * @param content
     * @param ext
     */
    public FastDFSFile(String name, byte[] content, String ext) {
        this.name = name;
        this.content = content;
        this.ext = ext;
    }

    public FastDFSFile(String name, byte[] content, String ext, String md5, String author) {
        this.name = name;
        this.content = content;
        this.ext = ext;
        this.md5 = md5;
        this.author = author;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public byte[] getContent() {
        return content;
    }

    public void setContent(byte[] content) {
        this.content = content;
    }

    public String getExt() {
        return ext;
    }

    public void setExt(String ext) {
        this.ext = ext;
    }

    public String getMd5() {
        return md5;
    }

    public void setMd5(String md5) {
        this.md5 = md5;
    }

    public String getAuthor() {
        return author;
    }

    public void setAuthor(String author) {
        this.author = author;
    }

util文件FastDFSUtil

package com.changgou.util;

import com.changgou.file.FastDFSFile;
import org.csource.common.NameValuePair;
import org.csource.fastdfs.*;
import org.springframework.core.io.ClassPathResource;

import java.io.ByteArrayInputStream;
import java.io.InputStream;

/**
 * @author ：musir
 */
public class FastDFSUtil {

    /**
     * 加载Tracker连接信息
     */
    static {
        try {
            //查询classpath下的文件路径
            String filename = new ClassPathResource("fdfs_client.conf").getPath();
            //载Tracker连接信息
            ClientGlobal.init(filename);
        } catch (Exception exception) {
            exception.printStackTrace();
        }
    }

    /**
     * 创建Tracker访问的客户端对象TrackerClient、通过TrackerClient访问TrackerServer服务，获取连接信息
     * @return
     * @throws Exception
     */
    public static TrackerServer getTrackerServer() throws Exception {
        TrackerClient trackerClient = new TrackerClient();
        TrackerServer trackerServer = trackerClient.getConnection();
        return trackerServer;
    }

    /**
     * 通过TrackerServer的连接信息可以获取Storage的连接信息，创建StorageClient对象存储的Storage的连接信息
     * @param trackerServer
     * @return
     * @throws Exception
     */
    public static StorageClient getStorageClient(TrackerServer trackerServer) throws Exception {
        StorageClient storageClient = new StorageClient(trackerServer, null);
        return storageClient;
    }

    /**
     * 文件上传，上传的文件的信息封装
     * @param fastDFSFile
     * @return
     * @throws Exception
     */
    public static String[] upload(FastDFSFile fastDFSFile) throws Exception {
        //附加参数
        NameValuePair[] meta_list = new NameValuePair[1];
        meta_list[0] = new NameValuePair("author", fastDFSFile.getAuthor());
        // 获取TrackerServer
        TrackerServer trackerServer = getTrackerServer();
        //获取StorageClient
        StorageClient storageClient = getStorageClient(trackerServer);
        //通过StorageCLient访问Storage，实现文件上传，并且获取文件上传后的存储信息，第一个参数:上传文件的字节数组、文件扩展名、附加参数
        String[] uploads = storageClient.upload_file(fastDFSFile.getContent(), fastDFSFile.getExt(), meta_list);
        return uploads;
    }

    /**
     * 获取文件信息
     * @param groupName 文件的组名
     * @param remoteFileName 文件的存储路径名字 M00/00/00/XXX.jpg
     * @return
     * @throws Exception
     */
    public static FileInfo getFile(String groupName, String remoteFileName) throws Exception {
        //获取Tracker
        TrackerServer trackerServer = getTrackerServer();
        //获取StorageClient
        StorageClient storageClient = getStorageClient(trackerServer);
        //获取文件信息
        return storageClient.get_file_info(groupName, remoteFileName);
    }

    /**
     * 文件下载
     * @param groupName
     * @param remoteFileName
     * @throws Exception
     */
    public static InputStream downloadFile(String groupName, String remoteFileName) throws Exception{
        // 获取TrackerServer
        TrackerServer trackerServer = getTrackerServer();
        // 获取StorageClient
        StorageClient storageClient = getStorageClient(trackerServer);
        // 文件下载
        byte[] buffer = storageClient.download_file(groupName, remoteFileName);
        return new ByteArrayInputStream(buffer);
    }

    /**
     * 文件删除
     * @param groupName
     * @param remoteFileName
     * @throws Exception
     */
    public static void deleteFile(String groupName, String remoteFileName) throws Exception{
        // 获取Tracker
        TrackerServer trackerServer = getTrackerServer();
        // 获取StorageClient
        StorageClient storageClient = getStorageClient(trackerServer);
        // 文件删除
        storageClient.delete_file(groupName, remoteFileName);
    }

    /**
     * 获取Storage信息
     * @throws Exception
     * @return
     */
    public static StorageServer getStorage() throws Exception{
        TrackerClient trackerClient = new TrackerClient();
        TrackerServer trackerServer = trackerClient.getConnection();
        // 获取Storage信息
        trackerClient.getStoreStorage(trackerServer);
        return trackerClient.getStoreStorage(trackerServer);
    }

    /**
     * 根据文件组名和文件存储路径获取Storage服务的IP、端口信息
     * @param groupName
     * @param remoteFileName
     * @throws Exception
     * @return
     */
    public static ServerInfo[] getServerInfo(String groupName, String remoteFileName) throws Exception{
        TrackerClient trackerClient = new TrackerClient();
        TrackerServer trackerServer = trackerClient.getConnection();
        // 获取Storage服务的IP、端口信息
        return trackerClient.getFetchStorages(trackerServer, groupName, remoteFileName);
    }

    /**
     * 获取Tracker的IP、服务端口
     * @throws Exception
     * @return
     */
    public static String getTrackerInfo() throws Exception{
        // 获取TrackerServer
        TrackerServer trackerServer = getTrackerServer();
        // 获取Tracker的IP、服务端口
        String hostString = trackerServer.getInetSocketAddress().getHostString();
        int tracker_http_port = ClientGlobal.getG_tracker_http_port();
        String url = "http://"+hostString+":"+tracker_http_port;
        return url;
    }
}

FileApplication启动类

package com.changgou;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.cloud.netflix.eureka.EnableEurekaClient;

/**
 * @author ：musir
 */
@SpringBootApplication(exclude = DataSourceAutoConfiguration.class)
@EnableEurekaClient
public class FileApplication {
    public static void main(String[] args) {
        SpringApplication.run(FileApplication.class, args);
    }
}

my-file-service-application.yml、fdfs_client.conf

spring:
  servlet:
    multipart:
      max-file-size: 10MB
      max-request-size: 10MB
  application:
    name: file
server:
  port: 18082
eureka:
  client:
    service-url:
      defaultZone: http://192.168.xxx.xxx:7001/eureka
  instance:
    prefer-ip-address: true
feign:
  hystrix:
    enabled: true

connect_timeout=60
network_timeout=60
charset=UTF-8
http.tracker_http_port=8080
tracker_server=192.168.xxx.xxx:22122