vue+springboot实现大文件上传

一、前情摘要

        最近开发的项目中需要上传一个1G左右的大文件，普通的文件上传方法无法满足这个需求，于是在网上找了一些相关资料后对整个文件上传的功能做了一些优化，通过对大文件分块上传的方式，实现了相关功能，包含了断点续传、秒传功能。

二、原先的实现

        该方法对于较小的文件是可以正常上传的，但是文件过大后会报内存溢出。

 public Map packetUpload(MultipartFile file) throws Exception {
       if (file == null) return createCommonPack(code300);
       File destFile = new File("/" + file.getOriginalFilename());
       file.transferTo(destFile);
       return createCommonPack(code200);
 }

三、修改后的实现

        处理文件，首先我们想到的是直接操作整个文件，但是对于大文件来说，直接操作明显不太现实，所有对于我们来说最简单的就是将大文件分割成一个个的小文件块，再去处理，将所有的文件块拿到后执行合并操作即可。

前端控制器：FileController.java

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartFile;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.util.Map;



/**
 * 大文件上传-前端控制器
 */
@RestController
@RequestMapping("/uploader")
@CrossOrigin
public class FileController extends ParentController {
    @Autowired
    private ChunkInfoService chunkInfoService;
    @Autowired
    private FileInfoService fileInfoService;
    @Autowired
    private RpcService rpcService;

    public static Integer msgid = 10;

    /**
     * 校验文件
     *
     * @param chunk
     * @param response
     * @return
     */
    @GetMapping("/chunk")
    public ChunkResult checkChunk(ChunkInfo chunk, HttpServletResponse response) {
        //logger.info("校验文件：{}", chunk);
        return chunkInfoService.checkChunkState(chunk, response);
    }

    /**
     * 文件块上传
     *
     * @param chunk
     * @return
     */
    @PostMapping("/chunk")
    public Integer uploadChunk(ChunkInfo chunk) {
        return chunkInfoService.uploadFile(chunk);
    }

    /**
     * 文件合并
     * @param fileInfo
     * @return
     * @throws Exception
     */
    @PostMapping("/mergeFile")
    public Map mergeFile(@RequestBody FileInfo fileInfo) throws Exception {
        //合并
        FileInfo resultfileInfo = fileInfoService.mergeFileSxx(fileInfo);
        if (resultfileInfo.getStatus() != HttpServletResponse.SC_OK ){
            return createCommonPack(code500, fileInfo);
        }
        return createCommonPack(code200, fileInfo);
    }

    //file 转换为 MultipartFile
    private MultipartFile getMulFileByPath(String filePath)
    {
        FileItemFactory factory = new DiskFileItemFactory(16, null);
        String textFieldName = "textField";
        int num = filePath.lastIndexOf(".");
        String extFile = filePath.substring(num);
        FileItem item = factory.createItem(textFieldName, "text/plain", true,
                "MyFileName" + extFile);
        File newfile = new File(filePath);
        int bytesRead = 0;
        byte[] buffer = new byte[8192];
        try
        {
            FileInputStream fis = new FileInputStream(newfile);
            OutputStream os = item.getOutputStream();
            while ((bytesRead = fis.read(buffer, 0, 8192))
                    != -1)
            {
                os.write(buffer, 0, bytesRead);
            }
            os.close();
            fis.close();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }

        MultipartFile mfile = new CommonsMultipartFile(item);
        return mfile;
    }

文件块service：ChunkInfoService.java

import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

@Service
@Slf4j
public class ChunkInfoService extends ParentSevice {
    
    /**
     * 校验当前文件
     * @param chunkInfo
     * @param response
     * @return 秒传or续传or新传
     */
    public ChunkResult checkChunkState(ChunkInfo chunkInfo, HttpServletResponse response) {
        ChunkResult chunkResult = new ChunkResult();
        String uploadFolder = CrashStatic.SYS_CONSOLE_YML.getProperty("sys_console.nbRuleFile.path");
        String file = uploadFolder + File.separator + chunkInfo.getIdentifier() + File.separator + chunkInfo.getFilename();
        if(FileInfoUtils.fileExists(file)) {
            chunkResult.setSkipUpload(true);
            chunkResult.setLocation(file);
            response.setStatus(HttpServletResponse.SC_OK);
            chunkResult.setMessage("完整文件已存在，执行秒传");
            return chunkResult;
        }
        List<Map> list = parentQuery.listBySql(
                "select chunk_number from t_chunk_info where identifier = '"+chunkInfo.getIdentifier()+"' and filename = '"+chunkInfo.getFilename()+"'");
        List<Integer> listNum = new ArrayList<>();
        if (null != list && list.size()>0){
            for (int i = 0; i < list.size(); i++) {
                String chunk_number = list.get(i).get("chunk_number").toString();
                int i1 = Long.valueOf(chunk_number).intValue();
                listNum.add(i1);
            }
        }
        if (list !=null && list.size() > 0) {
            chunkResult.setSkipUpload(false);
            chunkResult.setUploadedChunks(listNum);
            response.setStatus(HttpServletResponse.SC_OK);
            chunkResult.setMessage("部分文件块已存在，上传剩余文件块，断点续传");
            return chunkResult;
        }
        return chunkResult;
    }

    /**
     * 写文件
     * @param chunk
     * @return
     */
    public Integer uploadFile(ChunkInfo chunk) {
        Integer apiRlt = HttpServletResponse.SC_OK;
        MultipartFile file = chunk.getUpfile();
        try {
            byte[] bytes = file.getBytes();
            String uploadFolder = "你的路径";
            Path path = Paths.get(FileInfoUtils.generatePath(uploadFolder, chunk));
            Files.write(path, bytes);
            ChunkInfo save = parentModify.save(chunk);
            if(null != save){
                apiRlt = HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE;
            }
        } catch (IOException e) {
            log.error("写文件出错：{}",e.getMessage());
            apiRlt = HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE;
        }
        return apiRlt;
    }
}

操作文件service：FileInfoService.java

import org.springframework.stereotype.Service;

import javax.servlet.http.HttpServletResponse;
import java.io.File;

@Service
public class FileInfoService extends ParentSevice {

    /**
     * 文件合并
     * @param fileInfo
     * @return
     */
    public FileInfo mergeFileSxx(FileInfo fileInfo) {
        fileInfo.setStatus(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
        //进行文件的合并操作
        String filename = fileInfo.getFilename();
        String uploadFolder = "你的路径";
        String file = uploadFolder + File.separator + fileInfo.getIdentifier() + File.separator + filename;
        String folder = uploadFolder + File.separator + fileInfo.getIdentifier();
        Integer fileSuccess = FileInfoUtils.merge(file, folder, filename);
        fileInfo.setLocation(folder);
        parentModify.updateBySql("delete from t_chunk_info where identifier = '"+fileInfo.getIdentifier()+"' and filename = '"+fileInfo.getFilename()+"'");
        //文件合并成功后，更新状态
        if (fileSuccess == HttpServletResponse.SC_OK || fileSuccess == HttpServletResponse.SC_MULTIPLE_CHOICES) {
            fileInfo.setStatus(HttpServletResponse.SC_OK);
        }
        return fileInfo;
    }
}

操作文件的工具类：FileInfoUtils.java

import lombok.extern.slf4j.Slf4j;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.file.*;

/**
 * 操作文件工具类
 */
@Slf4j
public class FileInfoUtils {

    /**
     * 路径创建
     * @param uploadFolder
     * @param chunk
     * @return
     */
    public static String generatePath(String uploadFolder, ChunkInfo chunk) {
        StringBuilder sb = new StringBuilder();
        sb.append(uploadFolder).append("/").append(chunk.getIdentifier());
        //判断uploadFolder/identifier 路径是否存在，不存在则创建
        if (!Files.isWritable(Paths.get(sb.toString()))) {
            log.info("路径不存在，新建路径: {}", sb.toString());
            try {
                Files.createDirectories(Paths.get(sb.toString()));
            } catch (IOException e) {
                log.error("创建路径错误:{},{}",e.getMessage(), e);
            }
        }
        return sb.append("/")
                .append(chunk.getFilename())
                .append("-")
                .append(chunk.getChunkNumber()).toString();
    }

    /**
     * 文件合并
     * @param file
     * @param folder
     * @param filename
     * @return
     */
    public static Integer merge(String file, String folder, String filename){
        //默认合并成功
        Integer rlt = HttpServletResponse.SC_OK;
        try {
            //先判断文件是否存在
            if(fileExists(file)) {
                //文件已存在
                rlt = HttpServletResponse.SC_MULTIPLE_CHOICES;
            }else {
                //不存在的话，进行合并
                Files.createFile(Paths.get(file));
                Files.list(Paths.get(folder))
                        .filter(path -> !path.getFileName().toString().equals(filename))
                        .sorted((o1, o2) -> {
                            String p1 = o1.getFileName().toString();
                            String p2 = o2.getFileName().toString();
                            int i1 = p1.lastIndexOf("-");
                            int i2 = p2.lastIndexOf("-");
                            return Integer.valueOf(p2.substring(i2)).compareTo(Integer.valueOf(p1.substring(i1)));
                        })
                        .forEach(path -> {
                            try {
                                //追加写入文件
                                Files.write(Paths.get(file), Files.readAllBytes(path), StandardOpenOption.APPEND);
                                //合并后删除文件块
                                Files.delete(path);
                            } catch (IOException e) {
                                log.error("删除文件失败：{},{}",e.getMessage(), e);
                            }
                        });
            }
        } catch (IOException e) {
            log.error("合并失败：{},{}",e.getMessage(), e);
            //合并失败
            rlt = HttpServletResponse.SC_BAD_REQUEST;
        }
        return rlt;
    }

    /**
     * 根据文件的全路径名判断文件是否存在
     * @param file
     * @return
     */
    public static boolean fileExists(String file) {
        boolean fileExists = false;
        Path path = Paths.get(file);
        fileExists = Files.exists(path,new LinkOption[]{ LinkOption.NOFOLLOW_LINKS});
        return fileExists;
    }
}

实体类：ChunkInfo.java FileInfo.java

import java.time.LocalDateTime;
import lombok.Data;

import javax.persistence.*;

@Entity
@Table(name = "t_file_info")
@Data
@DataSourceDB(value = DB_MAIN, tableName = "文件")
public class FileInfo extends ParentDO {

    @FieldComment(value = "文件名")
    @Column(name = "filename")
    private String filename;

    @FieldComment(value = "文件标识")
    @Column(name = "identifier")
    private String identifier;

    @Transient
    private String uniqueIdentifier;

    @Transient
    private String name;

    @FieldComment(value = "总大小")
    @Column(name = "total_size")
    private Long totalSize;

    @Transient
    private String totalSizeName;

    @FieldComment(value = "存储地址")
    @Column(name = "location")
    private String location;

    @FieldComment(value = "文件类型")
    @Column(name = "filetype")
    private String filetype;

    @FieldComment(value = "文件所属")
    @Column(name = "ref_project_id")
    private String refProjectId;

    @FieldComment(value = "上传用户")
    @Column(name = "upload_user")
    private Integer uploadUser;

    @FieldComment(value = "上传时间")
    @Column(name = "upload_time")
    private LocalDateTime uploadTime;

    @Transient
    private int status;

    public void setTotalSize(Long totalSize) {
        this.totalSize = totalSize;
        if(1024*1024 > this.totalSize && this.totalSize >= 1024 ) {
            this.totalSizeName = String.format("%.2f",this.totalSize.doubleValue()/1024) + "KB";
        }else if(1024*1024*1024 > this.totalSize && this.totalSize >= 1024*1024 ) {
            this.totalSizeName = String.format("%.2f",this.totalSize.doubleValue()/(1024*1024)) + "MB";
        }else if(this.totalSize >= 1024*1024*1024 ) {
            this.totalSizeName = String.format("%.2f",this.totalSize.doubleValue()/(1024*1024*1024)) + "GB";
        }else {
            this.totalSizeName = this.totalSize.toString() + "B";
        }
    }
}

import lombok.Data;
import org.springframework.web.multipart.MultipartFile;

import javax.persistence.*;

@Entity
@Table(name = "t_chunk_info")
@Data
@DataSourceDB(value = DB_MAIN, tableName = "文件块")
public class ChunkInfo extends ParentDO {

    @FieldComment(value = "块编号，从1开始")
    @Column(name = "chunk_number")
    private Long chunkNumber;

    @FieldComment(value = "块大小")
    @Column(name = "chunk_size")
    private Long chunkSize;

    @FieldComment(value = "当前块大小")
    @Column(name = "current_chunk_size")
    private Long currentChunkSize;

    @FieldComment(value = "文件标识")
    @Column(name = "identifier")
    private String identifier;

    @FieldComment(value = "文件名")
    @Column(name = "filename")
    private String filename;

    @FieldComment(value = "相对路径")
    @Column(name = "relative_path")
    private String relativePath;

    @FieldComment(value = "总块数")
    @Column(name = "total_chunks")
    private Long totalChunks;

    @FieldComment(value = "总大小")
    @Column(name = "total_size")
    private Integer totalSize;

    @FieldComment(value = "文件类型")
    @Column(name = "file_type")
    private String fileType;

    @Transient
    private MultipartFile upfile;
}

四、前端代码

        前端代码请大家参考该链接

vue3+vue-simple-uploader +SpringBoot实现大文件分块上传-CSDN博客

五、总结

        本实现方案的核心思想是将一个大文件进行分割，对分割后的小文件上传之后进行合并请求处理，实现对大文件的处理，代码完整，拷贝即可，有缺失或者不明白的地方可以随时圈我，一起加油！