FASTDFS 文件系统分布式支持断点续传

最新推荐文章于 2022-10-25 13:41:23 发布

CnNoter

最新推荐文章于 2022-10-25 13:41:23 发布

阅读量1.6k

点赞数

分类专栏： Linux 后台笔记 VUE 文章标签： java fastdfs

本文链接：https://blog.csdn.net/weixin_38264066/article/details/118416700

版权

后台笔记同时被 3 个专栏收录

19 篇文章 0 订阅

订阅专栏

Linux

10 篇文章 0 订阅

订阅专栏

VUE

3 篇文章 0 订阅

订阅专栏

文件上传系统

最近公司在做分布式的文件存储系统,但问题出现了,文件系统使用哪一款?
最后决定用fastdfs在做,但是在springboot集成的时候出现了问题,前端使用vue组件在上传时候,因为并发比较高,直接将服务器给搞死了.原本打算将文件一块块的分片上传后再合并,然后在到fastdfs中,但集群的时候,文件上传到不同的服务器中.则没有办法在一个服务其中合并.所以最后决定将文件块直接向fastdfs中写,但是问题又来了,fastdfs创建一个文件,不能创建一块空间,只能在后边进行追加.这样的话,我们在服务器端,就不能多线程向里面跑数据了.因为必须要有顺序的上传.故此,前端只能先传第一块在传第二块,这样的传递.后端使用redis将文件信息保存,左锁机制.
最后其他服务调用我们oss服务需要将本地的一个文件保存到fastdfs中,openfeign 基本不支持,需要引入一些扩展包,进行处理,模拟表单.然后在上传.
一波三折啊.最后实现了.
下面是我的代码,高手别喷哦!!!^^

OSS系统代码
这是系统目录结构

import com.github.tobato.fastdfs.FdfsClientConfig;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableMBeanExport;
import org.springframework.context.annotation.Import;
import org.springframework.jmx.support.RegistrationPolicy;

@Configuration
@Import(FdfsClientConfig.class)
// Jmx重复注册bean的问题
@EnableMBeanExport(registration = RegistrationPolicy.IGNORE_EXISTING)
public class DfsConfig {
}

package com.techhero.component.oss.config;

import cn.hutool.core.io.FileUtil;
import com.github.tobato.fastdfs.domain.fdfs.StorePath;
import com.github.tobato.fastdfs.domain.proto.storage.DownloadByteArray;
import com.github.tobato.fastdfs.service.AppendFileStorageClient;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;
import java.io.*;
import java.nio.charset.Charset;

/**
 * 文件上传工具类
 */
@Slf4j
@Component
public class FastDFSClient {
    
    @Resource
    private FastFileStorageClient storageClient;
    
    @Resource
    private AppendFileStorageClient appendFileStorageClient;
    
    /**
     * 分片文件上传
     * @param group        文件上传组
     * @param inputStream  上传文件流
     * @param fileSize     当前块大小
     * @param fileName     文件名称
     * @param path         文件追加路径
     * @param offset       偏移量
     * @return
     */
    public String appendUpload(String group,InputStream inputStream,Long fileSize,String fileName,String path,Long offset){
        /*如果当前块是第一块则先添加文件*/
        if(StringUtils.isBlank(path)){
            StorePath storePath = appendFileStorageClient.uploadAppenderFile(group, inputStream, fileSize, FileUtil.extName(fileName));
            path  = storePath.getPath();
        }else{
            appendFileStorageClient.modifyFile(group, path, inputStream, fileSize,offset);
        }
        return path;
    }
    
    /*创建增量断点续传文件*/
    public String createAppendFile(String group,InputStream inputStream,Long fileSize,String fileName){
        StorePath storePath = appendFileStorageClient.uploadAppenderFile(group, inputStream, fileSize, FileUtil.extName(fileName));
        return storePath.getPath();
    }
    
    /**
     * 修改文件块
     * @param group 组名
     * @param path 路径
     * @param inputStream 文件流
     * @param fileSize 文件大小
     * @param offset 偏移量
     */
    public void modifyAppendFile(String group,String path,InputStream inputStream,Long fileSize,Long offset){
        appendFileStorageClient.modifyFile(group, path, inputStream, fileSize,offset);
    }
    
    /**
     * 上传文件
     */
    public String uploadImg(MultipartFile multipartFile) throws Exception {
        String originalFilename = multipartFile.getOriginalFilename().
                substring(multipartFile.getOriginalFilename().
                        lastIndexOf(".") + 1);
        StorePath storePath = this.storageClient.uploadImageAndCrtThumbImage(
                multipartFile.getInputStream(),
                multipartFile.getSize(), originalFilename, null);
        return storePath.getFullPath();
    }
    
    
    /**
     * 上传文件
     * @param multipartFile 文件对象
     * @return 文件访问地址
     * @throws IOException
     */
    public String uploadFile(MultipartFile multipartFile) throws IOException {
        String originalFilename = multipartFile.getOriginalFilename().
                substring(multipartFile.getOriginalFilename().
                        lastIndexOf(".") + 1);
        StorePath       storePath   = storageClient.uploadFile(multipartFile.getInputStream(),multipartFile.getSize(), originalFilename,null);
        return storePath.getFullPath();
    }
    
    /**
     * 上传文件
     * @param file 文件对象
     * @return 文件访问地址
     * @throws IOException
     */
    public String uploadFile(File file) throws IOException {
        FileInputStream inputStream = new FileInputStream (file);
        StorePath       storePath   = storageClient.uploadFile(inputStream,file.length(), FilenameUtils.getExtension(file.getName()),null);
        return storePath.getFullPath();
    }
    
    /**
     * 将一段字符串生成一个文件上传
     * @param content 文件内容
     * @param fileExtension
     * @return
     */
    public String uploadFile(String content, String fileExtension) {
        byte[]               buff      = content.getBytes(Charset.forName("UTF-8"));
        ByteArrayInputStream stream    = new ByteArrayInputStream(buff);
        StorePath            storePath = storageClient.uploadFile(stream,buff.length, fileExtension,null);
        return storePath.getFullPath();
    }
    
    /**
     * 下载文件
     *
     * @param fileUrl 文件URL
     * @return 文件字节
     * @throws IOException
     */
    public byte[] downloadFile(String fileUrl) throws IOException {
        if(fileUrl.startsWith("/")){
            fileUrl=fileUrl.substring(fileUrl.indexOf("/")+1);
        }
        String group = fileUrl.substring(0, fileUrl.indexOf("/"));
        String            path              = fileUrl.substring(fileUrl.indexOf("/") + 1);
        DownloadByteArray downloadByteArray = new DownloadByteArray();
        byte[]            bytes             = storageClient.downloadFile(group, path, downloadByteArray);
        return bytes;
    }
    
    /**
     * 删除文件
     */
    public Boolean deleteFile(String fileUrl) {
        if (StringUtils.isEmpty(fileUrl)) {
            log.info("fileUrl == >>文件路径为空...");
            return Boolean.FALSE;
        }
        try {
            StorePath storePath = StorePath.parseFromUrl(fileUrl);
            storageClient.deleteFile(storePath.getGroup(), storePath.getPath());
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("[文件服务-文件删除失败]", e);
            return Boolean.FALSE;
        }
    }
}

package com.techhero.component.oss.constant;

public interface UploadConstant {
    
    /*缓存过期时间*/
    long REDIS_EXPIRE = 60L * 60L * 12L;
    
    /*Redis存储KEY*/
    String FILE_UPLOAD = "FILE_UPLOAD:";
    
    /*当前文件路径*/
    String FILE_UPLOAD_PATH = FILE_UPLOAD + "PATH:";
    
    /*当前文件已上传大小*/
    String FILE_UPLOAD_SIZE = FILE_UPLOAD + "SIZE:";
    
    /*当前最大块号*/
    String FILE_UPLOAD_CHUNK = FILE_UPLOAD + "CHUNK:";
    
}

package com.techhero.component.oss.controller;

import com.techhero.common.model.oss.MultipartFileChunk;
import com.techhero.common.utils.req.ResBean;
import com.techhero.component.oss.service.FileUploadService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;

@Slf4j
@RestController
@RequestMapping("big")
public class BigFileUploadController {
    
    @Resource
    private FileUploadService fileUploadService;
   
    
    /**
     * 分片文件上传
     */
    @PostMapping("trunkUpload")
    public ResBean trunkUpload(MultipartFileChunk multipartFileChunk){
        return fileUploadService.trunkUpload(multipartFileChunk);
    }
    
    /**
     * 分片文件上传
     */
    @PostMapping(value = "feignUpload",consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
    public ResBean feignUpload(MultipartFile file,
                               String identifier,
                               Long chunkSize,
                               Long totalChunks,
                               Long totalSize,
                               Long chunkNumber,
                               Long currentChunkSize,
                               String fileName
                               ){
        return fileUploadService.trunkUpload(new MultipartFileChunk(identifier,chunkSize,totalChunks,totalSize,chunkNumber,currentChunkSize,fileName,file));
    }
    
    /**
     * 获取文件路径
     */
    @GetMapping("getFilePath")
    public ResBean getFilePath(@RequestParam String identifier){
        return fileUploadService.getFilePath(identifier);
    }


}

package com.techhero.component.oss.controller;

import com.techhero.common.model.oss.MultipartFileChunk;
import com.techhero.common.utils.req.ResBean;
import com.techhero.component.oss.service.FileUploadService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.tomcat.util.http.fileupload.servlet.ServletFileUpload;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;

/**
 * 文件上传接口
 */
@Slf4j
@RestController
@RequestMapping("small")
public class SmallFileController {
    
    /*上传文件位置*/
    @Value("${fdfs.file.path}")
    private String filePath;
    
    /*文件临时目录*/
    @Value("${fdfs.file.tempPath}")
    private String filePathTemp;
    
    @Resource
    private FileUploadService fileUploadService;
    
    /**
     * 文件上传
     * http://xxx/group1/M00/00/00/wKhIgl0n4AKABxQEABhlMYw_3Lo825.png
     */
    @PostMapping(value = "/uploadFile")
    public ResBean uploadFile(@RequestParam("file") MultipartFile file) {
        return fileUploadService.uploadFile(file);
    }
    
    /**
     * 文件删除
     */
    @GetMapping(value = "/deleteByPath")
    public ResBean deleteByPath(@RequestParam("filePath") String filePath) {
        return fileUploadService.delFileByPath(filePath);
    }
    
    /**
     * 文件下载
     * @param filePath 文件路径
     */
    @GetMapping("/download")
    public void downloadFile(@RequestParam("filePath") String filePath, HttpServletResponse response) throws Exception {
        fileUploadService.downloadFile(filePath,response);
    }
   
    
    /**
     * 分片上传(大文件)
     */
    @PostMapping("upload")
    public ResBean upload(MultipartFileChunk chunk, HttpServletRequest request) throws IOException {
        
        boolean isMultipart = ServletFileUpload.isMultipartContent(request);
        if (isMultipart) {
            MultipartFile file = chunk.getFile();
            
            if (file == null) {
                return ResBean.failed("文件未选择");
            }
            
            Long chunkNumber = chunk.getChunkNumber();
            if (chunkNumber == null) {
                chunkNumber = 0L;
            }
            
            File outFile = new File(filePathTemp + File.separator + chunk.getIdentifier(), chunkNumber + ".part");
    
            if(outFile.exists()){
                return ResBean.success("附件上传成功");
            }
            
            InputStream inputStream = file.getInputStream();
            FileUtils.copyInputStreamToFile(inputStream, outFile);
            return ResBean.success("附件上传成功");
        }
        
        return ResBean.failed("没有文件");
    }
    
    /**
     * 合并所有分片
     */
    @GetMapping("/merge")
    @ResponseBody
    public ResBean merge(String fileName, String identifier) throws Exception {
        
        File file = new File(filePathTemp + File.separator + identifier);
        if (file.isDirectory()) {
            File[] files = file.listFiles();
            if (files != null && files.length > 0) {
                File partFile = new File(filePath + File.separator + fileName);
                for (int i = 1; i <= files.length; i++) {
                    File s = new File(filePathTemp + File.separator + identifier, i + ".part");
                    try (FileOutputStream destTempFos = new FileOutputStream(partFile, true)) {
                        FileUtils.copyFile(s, destTempFos);
                    }
                }
                //删除临时目录文件
                FileUtils.deleteDirectory(file);
                return ResBean.success(fileName);
            }
        }
        return ResBean.failed("文件上传失败");
    }
    
}

package com.techhero.component.oss.service;

import com.github.tobato.fastdfs.domain.conn.FdfsWebServer;
import com.techhero.common.bean.config.RedisCacheService;
import com.techhero.common.model.oss.MultipartFileChunk;
import com.techhero.common.utils.req.ResBean;
import com.techhero.component.oss.config.FastDFSClient;
import com.techhero.component.oss.constant.UploadConstant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;

@Slf4j
@Service
public class FileUploadService {
    
    /**
     * 解析路径
     */
    private static final String SEPARATOR = "/";
    
    @Value("${fdfs.groupName:group1}")
    private String groupName;
    
    /**
     * 文件上传客户端
     */
    @Resource
    private FastDFSClient fastDFSClient;
    
    /**
     * 缓存
     */
    @Resource
    private RedisCacheService redisCacheService;
    
    /**
     * web地址配置类
     */
    @Resource
    private FdfsWebServer fdfsWebServer;
    
    /**
     * 简单文件上传
     *
     * @param file 文件
     */
    public ResBean uploadFile(MultipartFile file) {
        try {
            String path = fastDFSClient.uploadFile(file);
            if (!org.springframework.util.StringUtils.isEmpty(path)) {
                if (!path.startsWith("/")) {
                    return ResBean.success("/" + path);
                }
                return ResBean.success(path);
            } else {
                log.info("[文件服务-文件上传失败]@路径为空");
            }
        } catch (Exception e) {
            log.error("[文件上传失败]", e);
        }
        return ResBean.failed("网络正忙,请重试...");
    }
    
    /**
     * 文件路径
     *
     * @param filePath 文件路径
     */
    public ResBean delFileByPath(String filePath) {
        log.info("[文件删除]@ {}", filePath);
        Boolean delFlag = fastDFSClient.deleteFile(filePath);
        return ResBean.validCountBean(delFlag);
    }
    
    /**
     * 文件下载
     *
     * @param filePath 文件路径
     */
    public void downloadFile(String filePath, HttpServletResponse response) throws Exception {
        String fileName = filePath.substring(filePath.lastIndexOf(".") + 1);
        log.debug("[文件下载]-[文件名称={}]", fileName);
        byte[] bytes = fastDFSClient.downloadFile(filePath);
        response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(fileName, "UTF-8"));
        response.setCharacterEncoding("UTF-8");
        try (ServletOutputStream outputStream = response.getOutputStream()) {
            outputStream.write(bytes);
            outputStream.flush();
        } catch (Exception e) {
            log.error("[文件下载-异常]", e);
        }
    }
    
    /**
     * 分片文件上传
     *
     * @param multipartFileChunk 分片文件对象
     * @describe 返回路径 M00/00/00/rBID22Da54CEOok0AAAAAJscELM782.exe
     */
    public ResBean trunkUpload(MultipartFileChunk multipartFileChunk) {
        log.info("[分片文件上传-上传数据开始]#\n{}",multipartFileChunk);
        MultipartFile multipartFile = multipartFileChunk.getFile();
        /*总块数*/
        Long totalChunks = multipartFileChunk.getTotalChunks();
        /*当前块角标*/
        Long chunkNumber = multipartFileChunk.getChunkNumber();
        /*文件唯一凭证信息*/
        String identifier = multipartFileChunk.getIdentifier();
        /*当前块大小*/
        Long currentChunkSize = multipartFile.getSize();
        
        /*第一块文件路径标记*/
        String filePath = redisCacheService.get(UploadConstant.FILE_UPLOAD_PATH + identifier);
        
        try (InputStream inputStream = multipartFile.getInputStream()) {
            if (chunkNumber.equals(1L) && filePath == null) {
                /*文件名称*/
                String fileName = multipartFileChunk.getFileName();
                
                String path = fastDFSClient.createAppendFile(groupName, inputStream, currentChunkSize, fileName);
                
                /*存放路径*/
                redisCacheService.set(UploadConstant.FILE_UPLOAD_PATH + identifier, path, UploadConstant.REDIS_EXPIRE);
                /*存放当前文件大小*/
                redisCacheService.set(UploadConstant.FILE_UPLOAD_SIZE + identifier, currentChunkSize.toString(), UploadConstant.REDIS_EXPIRE);
                /*最大块号*/
                redisCacheService.set(UploadConstant.FILE_UPLOAD_CHUNK + identifier, chunkNumber.toString(), UploadConstant.REDIS_EXPIRE);
                return ResBean.success(path);
            }
            
            
            /*-----------下面是文件已经创建,冲第二块开始,则走下边流程------------*/
            
            /*如果第一块没有上传,则其他块不允许上传*/
            if (chunkNumber > 1L && StringUtils.isBlank(filePath)) {
                return ResBean.failed("请等待头文件传输完毕,文件马上上传...");
            }
            
            /*文件最大块号*/
            String fileUploadChunk    = redisCacheService.get(UploadConstant.FILE_UPLOAD_CHUNK + identifier);
            Long   fileUploadChunkMax = Long.parseLong(fileUploadChunk) + 1L;
            if (chunkNumber > 1L
                    && StringUtils.isNotBlank(filePath)
                    && StringUtils.isNotBlank(fileUploadChunk)
                    && chunkNumber.equals(fileUploadChunkMax)) {
                /*文件已上传大小*/
                String fileUploadSizeRedis = redisCacheService.get(UploadConstant.FILE_UPLOAD_SIZE + identifier);
                Long   fileUploadSize      = Long.valueOf(fileUploadSizeRedis);
                fastDFSClient.modifyAppendFile(groupName, filePath, inputStream, currentChunkSize, fileUploadSize);
                
                long historyFileSize = currentChunkSize + fileUploadSize;
                /*当前文件已经上传大小*/
                redisCacheService.set(UploadConstant.FILE_UPLOAD_SIZE + identifier, Long.toString(historyFileSize), UploadConstant.REDIS_EXPIRE);
                /*最大块号*/
                redisCacheService.set(UploadConstant.FILE_UPLOAD_CHUNK + identifier, chunkNumber.toString(), UploadConstant.REDIS_EXPIRE);
                /*如果当前块角标与总块数相等或者大于的情况下则,设定redis设定一下超时时间,防止用户页面跳转后一直没有清理缓存*/
                if (chunkNumber >= totalChunks) {
                    /*这里只需要设定路径缓存即可,因为路径缓存超时时间设定比较早而其他缓存则刚设置,延迟不到一秒,没有并发则忽略延迟*/
                    redisCacheService.expire(UploadConstant.FILE_UPLOAD_PATH, UploadConstant.REDIS_EXPIRE);
                }
                
                return ResBean.success(filePath);
            }
            
        } catch (IOException e) {
            log.error("[分片文件上传-产生流异常]", e);
        }
        return ResBean.failed("当前文件等待上传...");
    }
    
    /**
     * 获取文件路径
     * @param identifier 文件唯一凭证
     */
    public ResBean getFilePath(String identifier) {
        String filePath = redisCacheService.get(UploadConstant.FILE_UPLOAD_PATH + identifier);
        String[] keys = new String[]{
                UploadConstant.FILE_UPLOAD_PATH + identifier,//路径
                UploadConstant.FILE_UPLOAD_SIZE + identifier,//已上传文件大小
                UploadConstant.FILE_UPLOAD_CHUNK + identifier,//当前已上传文件块
        };
        /*清除redis缓存*/
        redisCacheService.del(keys);
        String fullPath = getFullPath(fdfsWebServer.getWebServerUrl(), groupName, filePath);
        return ResBean.success(fullPath);
    }
    
    /*拼接文件访问路径*/
    private String getFullPath(String webUrl, String groupName, String filePath) {
        return (webUrl + SEPARATOR + groupName + SEPARATOR + filePath).replace(SEPARATOR.concat(SEPARATOR), SEPARATOR);
    }
    
}

server:
  port: 19908
spring:
  autoconfigure:
    exclude: org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration,org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration
  application:
    name: techhero-component-oss
  profiles:
    active: local
  cloud:
    config:
      fail-fast: true
      discovery:
        serviceId: techhero-config-server
        enabled: true
      profile: ${spring.profiles.active}
      label: ${spring.profiles.active}
security:
  basic:
    enabled: false


---
spring:
  profiles: local
eureka:
  instance:
    prefer-ip-address: true
    lease-renewal-interval-in-seconds: 5
    lease-expiration-duration-in-seconds: 20
  client:
    serviceUrl:
      defaultZone: http://techhero:techhero123@127.0.0.1:10421/eureka
    registry-fetch-interval-seconds: 10

<!-- FastDFS依赖 -->
        <dependency>
            <groupId>com.github.tobato</groupId>
            <artifactId>fastdfs-client</artifactId>
            <version>1.26.5</version>
        </dependency>
 <!--连接池-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid</artifactId>
            <version>${druid.version}</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>${druid.starter.version}</version>
        </dependency>
        <!--FEIGN -->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-feign</artifactId>
        </dependency>
        <!-- 消息总线 -->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-bus-amqp</artifactId>
        </dependency>
        <!--这个包下面会沾-->
        <dependency>
            <groupId>com.techhero.common</groupId>
            <artifactId>techhero-common</artifactId>
            <version>1.0-SNAPSHOT</version>
        </dependency>

common包依赖

<!--Redis-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
        </dependency>
        <!--添加spring对cache的支持-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-cache</artifactId>
        </dependency>


        <!--消息总线-->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-bus-amqp</artifactId>
        </dependency>
        <!--JWT-->
        <dependency>
            <groupId>io.jsonwebtoken</groupId>
            <artifactId>jjwt</artifactId>
            <version>${jjwt.version}</version>
        </dependency>
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus</artifactId>
            <version>${mybatis-plus.version}</version>
        </dependency>
        <dependency>
            <groupId>com.xiaoleilu</groupId>
            <artifactId>hutool-all</artifactId>
            <version>${hutool.version}</version>
        </dependency>

        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.0.6</version>
        </dependency>
        <!-- TTL -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>transmittable-thread-local</artifactId>
            <version>${ttl.version}</version>
        </dependency>
        <!--切面增强AOP-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-aop</artifactId>
        </dependency>
        <!-- 序列化工具 -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.46</version>
        </dependency>
        <dependency>
            <groupId>org.codehaus.jackson</groupId>
            <artifactId>jackson-mapper-asl</artifactId>
            <version>1.9.13</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.3.0</version>
        </dependency>
        <!-- okhttp -->
        <dependency>
            <groupId>io.github.openfeign</groupId>
            <artifactId>feign-okhttp</artifactId>
        </dependency>
        <!--<dependency>
            <groupId>com.netflix.feign</groupId>
            <artifactId>feign-okhttp</artifactId>
            <version>8.18.0</version>
        </dependency>-->

        <!--swagger-->
        <dependency>
            <groupId>io.springfox</groupId>
            <artifactId>springfox-swagger2</artifactId>
            <version>${swagger2.version}</version>
        </dependency>
        <!--配置属性加密工具-->
        <dependency>
            <groupId>com.github.ulisesbocchio</groupId>
            <artifactId>jasypt-spring-boot-starter</artifactId>
            <version>${jasypt.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-examples</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-excelant</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-ooxml</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-ooxml-schemas</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-scratchpad</artifactId>
            <version>${apache.poi.version}</version>
        </dependency>

        <dependency>
            <groupId>commons-fileupload</groupId>
            <artifactId>commons-fileupload</artifactId>
            <version>${fileupload.version}</version>
        </dependency>
        <!-- spring-boot-starter-data-mongodb -->
        <!--<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-mongodb</artifactId>
        </dependency>-->
        <!-- 邮件 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-mail</artifactId>
        </dependency>
        <!--security -->
        <!--<dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-core</artifactId>
            <version>4.2.5.RELEASE</version>
        </dependency>-->
        <dependency>
            <groupId>com.netflix.zuul</groupId>
            <artifactId>zuul-core</artifactId>
            <version>2.1.3</version>
            <scope>compile</scope>
        </dependency>
        <!--TechHero-->
<!--        <dependency>-->
<!--            <groupId>com.itechhero</groupId>-->
<!--            <artifactId>module-framework-cache</artifactId>-->
<!--            <version>1.0.1-RC1</version>-->
<!--        </dependency>-->
<!--        <dependency>-->
<!--            <groupId>com.itechhero</groupId>-->
<!--            <artifactId>module-framework-core</artifactId>-->
<!--            <version>1.0.1-RC1</version>-->
<!--        </dependency>-->
        <!--通用Mapper-->
        <dependency>
            <groupId>tk.mybatis</groupId>
            <artifactId>mapper</artifactId>
            <version>${tkmapper.version}</version>
        </dependency>
        <!--数据库驱动-->
        <dependency>
            <groupId>com.oracle</groupId>
            <artifactId>ojdbc6</artifactId>
            <version>${oracle.version}</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.46</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.mybatis.generator</groupId>
            <artifactId>mybatis-generator-core</artifactId>
            <version>1.3.6</version>
        </dependency>

        <!--<dependency>
            <groupId>com.techhero.common</groupId>
            <artifactId>techhero-common-cache</artifactId>
            <version>1.0-SNAPSHOT</version>
        </dependency>-->

        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>commons-beanutils</groupId>
            <artifactId>commons-beanutils</artifactId>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>net.sf.ezmorph</groupId>
            <artifactId>ezmorph</artifactId>
            <version>1.0.6</version>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>net.sf.json-lib</groupId>
            <artifactId>json-lib</artifactId>
            <version>2.4</version>
            <type>jar</type>
            <classifier>jdk15</classifier>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.7</version>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>org.apache.velocity</groupId>
            <artifactId>velocity</artifactId>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-test</artifactId>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>dom4j</groupId>
            <artifactId>dom4j</artifactId>
            <version>1.6.1</version>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>org.springframework.security</groupId>
            <artifactId>spring-security-core</artifactId>
        </dependency>
        <!--原core包中所用到的依赖-->
        <dependency>
            <groupId>commons-httpclient</groupId>
            <artifactId>commons-httpclient</artifactId>
            <version>3.1</version>
        </dependency>
        <dependency>
            <groupId>com.github.pagehelper</groupId>
            <artifactId>pagehelper</artifactId>
            <version>4.1.1</version>
            <scope>compile</scope>
        </dependency>

        <!--quartz相关依赖-->
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz</artifactId>
            <version>2.3.0</version>
        </dependency>
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz-jobs</artifactId>
            <version>2.3.0</version>
        </dependency>

        <dependency>
            <groupId>io.github.openfeign.form</groupId>
            <artifactId>feign-form</artifactId>
            <version>3.0.3</version>
        </dependency>
        <dependency>
            <groupId>io.github.openfeign.form</groupId>
            <artifactId>feign-form-spring</artifactId>
            <version>3.0.3</version>
        </dependency>

下边是common包写的类,这个是为了其他子工程不需要做任何事儿,只需要依赖于common工程就可以直接feign调用oss系统

用到的类

package com.techhero.common.bean.config.oss;

import com.techhero.common.feigin.OssFeign;
import com.techhero.common.utils.JSONUtil;
import com.techhero.common.utils.db.IdWorker;
import com.techhero.common.utils.req.ResBean;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartFile;

import javax.annotation.Resource;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;

/**
 * 全局提供文件上传工具类
 */
@Slf4j
@Component
public class OssClientUtils {
    
    @Resource
    private OssFeign ossFeign;
    
    public static void main(String[] args) {
        long a = 18 / 19;
        System.out.println(a);
    }
    
    /**
     * 分片文件上传
     * @param file 文件对象
     * @param splitSize 分块上传每块多大 1M=1*1024*1024
     * @param retryNum 重试次数(文件上传时候出现上传失败情况,这种情况下,重试多少次)
     * @return 返回响应对象
     */
    public ResBean upload(File file, Long splitSize, int retryNum) {
        String fileName = file.getName();
        
        /*文件总大小*/
        Long fileTotalSize = file.length();
        
        long totalChunks = fileTotalSize / splitSize;
        
        if (fileTotalSize % splitSize > 0 || totalChunks == 0) {
            totalChunks++;
        }
        /*文件唯一标识*/
        String          identifier = IdWorker.getOrderIdByUUId();
        FileItemFactory factory    = new DiskFileItemFactory(16, null);
        
        try (FileInputStream in = new FileInputStream(file)) {
            for (int i = 1; i <= totalChunks; i++) {
                FileItem     item = factory.createItem("file", "multipart/form-data", true, fileName);
                try(OutputStream out = item.getOutputStream()){
                    byte[] b    = new byte[splitSize.intValue()];
                    int    read = in.read(b);
    
                    log.debug("[读取字节长度{}-byte]", read);
                    out.write(b, 0, read);
                    out.flush();
                    MultipartFile multipartFile = new CommonsMultipartFile(item);
                    
                    /*重试机制*/
                    for (int r = 0; r < retryNum; r++) {
                        ResBean resBean = ossFeign.feignUpload(multipartFile, identifier, splitSize, totalChunks, fileTotalSize, (long) i, (long) read, fileName);
                        log.info("[FEIGN文件上传,返回信息]#{}", JSONUtil.objectToJson(resBean));
                        if (resBean.isSuccess()) {
                            log.info("[FEIGN文件上传,上传成功]#{}", resBean.getData());
                            break;
                        }
                        /*如果是最后一次,则返回上传失败*/
                        if (r == retryNum - 1) {
                            return ResBean.failed("[文件上传][" + i + "块上传" + retryNum + "次不成功..]");
                        }
                    }
                }
            }
            ResBean endResBean = ossFeign.getFilePath(identifier);
            if (endResBean.isSuccess()) {
                log.info("[FEIGN文件上传,**文件切片全部**上传成功]#{}", endResBean.getData());
                return endResBean;
            }
            
        } catch (IOException e) {
            log.error("[切割文件FEIGN调用-出现IO异常]", e);
            return ResBean.failed("出现IO异常");
        }
        
        return ResBean.failed("文件上传失败...");
    }
    
}

package com.techhero.common.feigin;

import feign.codec.Encoder;
import com.techhero.common.bean.config.oss.MultipartSupportConfig;
import com.techhero.common.constant.ServiceNameConstant;
import com.techhero.common.feigin.fallback.OssFeignFallback;
import com.techhero.common.utils.req.ResBean;
import feign.form.spring.SpringFormEncoder;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.web.HttpMessageConverters;
import org.springframework.cloud.netflix.feign.FeignClient;
import org.springframework.cloud.netflix.feign.support.SpringEncoder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

@FeignClient(path = "big", value = ServiceNameConstant.OSS_SERVICE, fallback = OssFeignFallback.class, configuration = OssFeign.MultipartSupportConfig.class)
public interface OssFeign {
    
       /*@PostMapping(value="trunkUpload",consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
    ResBean trunkUpload(@RequestBody MultipartFileChunk multipartFileChunk);*/
    
    /**
     * 分片文件上传
     *
     * @param identifier       唯一标识
     * @param chunkSize        分片大小
     * @param totalChunks      总分片数量
     * @param totalSize        总大小
     * @param chunkNumber      往前分片号
     * @param currentChunkSize 当前分片大小
     * @param fileName         文件名称
     * @param file             文件
     */
    @PostMapping(value = "feignUpload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
    ResBean feignUpload(
            @RequestPart("file") MultipartFile file,
            @RequestParam("identifier") String identifier,
            @RequestParam("chunkSize") Long chunkSize,
            @RequestParam("totalChunks") Long totalChunks,
            @RequestParam("totalSize") Long totalSize,
            @RequestParam("chunkNumber") Long chunkNumber,
            @RequestParam("currentChunkSize") Long currentChunkSize,
            @RequestParam("fileName") String fileName
    );
    
    /**
     * 获取文件路径
     *
     * @param identifier 文件唯一标识
     */
    @GetMapping("getFilePath")
    ResBean getFilePath(@RequestParam("identifier") String identifier);
    
    @Configuration
    class MultipartSupportConfig {
        
        @Autowired
        private ObjectFactory<HttpMessageConverters> messageConverters;
        
        @Bean
        public Encoder feignFormEncoder() {
            return new SpringFormEncoder(new SpringEncoder(messageConverters));
        }
    }
}

package com.techhero.common.feigin.fallback;

import com.techhero.common.feigin.OssFeign;
import com.techhero.common.utils.req.ResBean;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

@Slf4j
@Service
public class OssFeignFallback implements OssFeign {
    
    /**
     * @param identifier       唯一标识
     * @param chunkSize        分片大小
     * @param totalChunks      总分片数量
     * @param totalSize        总大小
     * @param chunkNumber      往前分片号
     * @param currentChunkSize 当前分片大小
     * @param fileName         文件名称
     * @param file             文件
     */
    @Override
    public ResBean feignUpload(
            @RequestPart("file") MultipartFile file,
            @RequestParam("identifier") String identifier,
            @RequestParam("chunkSize") Long chunkSize,
            @RequestParam("totalChunks") Long totalChunks,
            @RequestParam("totalSize") Long totalSize,
            @RequestParam("chunkNumber") Long chunkNumber,
            @RequestParam("currentChunkSize") Long currentChunkSize,
            @RequestParam("fileName") String fileName
            ) {
        log.info("[文件上传->分片上传]#远程调用失败&\n{}", identifier);
        return ResBean.failed("[文件上传->分片上传]#远程调用失败,请重试..");
    }
    
    /**
     * 获取文件路径
     *
     * @param identifier 文件唯一标识
     */
    @Override
    public ResBean getFilePath(String identifier) {
        log.info("[文件上传-获取文件全路径]#远程调用失败&\n文件唯一标识={}", identifier);
        return ResBean.failed("[文件上传-获取文件全路径]#远程调用失败,请重试..");
    }
}

package com.techhero.common.model.oss;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.springframework.web.multipart.MultipartFile;

import java.io.Serializable;

@Data
@ToString
@AllArgsConstructor
@NoArgsConstructor
public class MultipartFileChunk implements Serializable {
    /*文件标识*/
    private String identifier;
    
    /*分块大小*/
    private Long chunkSize;
    
    /*当前文件总块数*/
    private Long totalChunks;
    
    /*当前文件总大小*/
    private Long totalSize;
    
    /*当前块号*/
    private Long chunkNumber;
    
    /*当前块大小*/
    private Long currentChunkSize;
    
    /*文件名称*/
    private String fileName;
    
    /*文件*/
    private MultipartFile file;
    
}

OK 目前上面的代码就是如此,下面是子工程调用的地方顺便搞出来吧
只需要把OssClientUtils注入到业务类里面即可

package com.techhero.base.transdata.service;


import com.techhero.common.bean.config.oss.OssClientUtils;
import com.techhero.common.utils.JSONUtil;
import com.techhero.common.utils.req.ResBean;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.File;

@Slf4j
@Component("fileUpLoadService")
public class FileUpLoadServiceImpl {
    
    @Resource
    private OssClientUtils ossClientUtils;
    
    //@PostConstruct
    public void load(){
        File file=new File("/Users/cnnoter/Downloads/21_06_26 08_42_34.mp4");
        //第一个参数文件对象,第二个参数分片大小这个是10M分片大小,最后边是重试次数
        ResBean upload = ossClientUtils.upload(file, 10 * 1024 * 1024L, 5);
        log.info("传输完成\n{}", JSONUtil.objectToJson(upload));
    }
    
}

前端代码

<template>
  <div>
     <el-button icon="el-icon-circle-plus" type="primary" @click="start">打开上传文件框</el-button>
     <el-dialog
    title="上传"
    :visible.sync="dialogVisible"
    width="50%"
    class="el-dialog-global"
    >
    <div class="sec">
      <uploader :options="options" :file-status-text="statusText" ref="uploader" @file-complete="fileComplete" @complete="complete"></uploader>
    </div>
    <div slot="footer" class="dialog-footer">
      <el-button @click="dialogVisible = false">取 消</el-button>
      <el-button type="primary" @click="dialogVisible = false">确 定</el-button>
    </div>
  </el-dialog>
  </div>
  
</template>

<script>
  import axios from 'axios';
  export default {
    data () {
      return {
        dialogVisible: false,
        options: {
          target: axios.defaults.baseURL + '/oss/big/trunkUpload',
          testChunks: false,
          maxChunkRetries: 1000, //最大自动失败重试上传次数
         simultaneousUploads:1,
          chunkSize: 10 * 1024 * 1024,//分片大小
          // checkChunkUploadedByResponse: function (chunk, message) {
          //   let objMessage = JSON.parse(message);
          //   console.log(objMessage);
          //   console.log(objMessage.success==true);
          //    return objMessage.success==true;
          // },
          processParams(params) {//每一次分片传给后台的参数，params是该方法返回的形参，包含分片信息
            console.log(params, '123');
            params.fileName=params.filename;
            return params;
            // return {//返回一个对象，会添加到每一个分片的请求参数里面
            //   chunkSize: params.chunkSize,
            //   totalSize: params.totalSize,
            //   filename: params.filename,
            //   identifier: params.identifier,
            //   totalChunks: params.totalChunks,
            //   chunkNumber: params.chunkNumber,
            // };
          }
        },
        attrs: {
          accept: 'image/*'
        },
        statusText: {
          success: '成功了',
          error: '出错了',
          uploading: '上传中',
          paused: '暂停中',
          waiting: '等待中'
        }
      }
    },
    methods: {
      start(){
        this.dialogVisible=true;
      },
      complete () {
        console.log('complete', arguments)
      },
      fileComplete () {
        console.log('file complete', arguments)
        const file = arguments[0].file
        let url = '/big/getFilePath?identifier=' + arguments[0].uniqueIdentifier
        this.$httpclient.get(url, {}, res => {
          console.log(res);
        }, 'oss')
      }
    },
    mounted () {
      this.$nextTick(() => {
        window.uploader = this.$refs.uploader.uploader
      })
    }
  }
</script>

CnNoter

关注

0
点赞
踩
10

收藏

觉得还不错? 一键收藏
0
评论
FASTDFS 文件系统分布式支持断点续传

文件上传系统最近公司在做分布式的文件存储系统,但问题出现了,文件系统使用哪一款?最后决定用fastdfs在做,但是在springboot集成的时候出现了问题,前端使用vue组件在上传时候,因为并发比较高,直接将服务器给搞死了.原本打算将文件一块块的分片上传后再合并,然后在到fastdfs中,但集群的时候,文件上传到不同的服务器中.则没有办法在一个服务其中合并.所以最后决定将文件块直接向fastdfs中写,但是问题又来了,fastdfs创建一个文件,不能创建一块空间,只能在后边进行追加.这样的话,我们在
复制链接

扫一扫