JAVA网页上传大文件时如何处理断点续传？

2501_90646763

于 2025-10-04 13:56:30 发布

阅读量512

点赞数 12

CC 4.0 BY-SA版权

文章标签： java 开发语言 java大文件上传 java大文件秒传 java大文件上传下载 java文件传输解决方案 jsp大文件上传

本文链接：https://blog.csdn.net/2501_90646763/article/details/152511954

大文件传输系统解决方案

作为浙江IT行业软件公司项目负责人，我们面临的大文件传输需求具有很高的技术挑战性。以下是我针对该需求的专业解决方案分析。

需求分析总结

超大文件传输：单文件100GB，文件夹层级结构保持
高稳定性：支持断点续传(浏览器刷新/关闭不丢失进度)
安全要求：支持SM4、AES加密传输与存储，自动解密下载
高性能：非打包下载方案(解决服务器内存问题)
兼容性：多平台(Windows 7+/macOS/Linux)、多浏览器(含IE8)
技术栈：兼容JSP/Spring Boot/Vue2/Vue3/React
部署：支持阿里云OSS/ECS，内网公网部署
授权模式：买断授权(预算98万内)

技术方案设计

系统架构

[客户端] --> [负载均衡] --> [Web服务器] --> [应用服务器] --> [数据库]
                           ↑
                           ↓
                      [文件存储]
                      (阿里云OSS/本地存储)

前端实现方案

// Vue2示例 - 大文件上传组件



import { encryptFile, generateFileKey } from '@/utils/crypto';
import { uploadFile } from '@/api/file';

export default {
  data() {
    return {
      files: [],
      uploader: null
    };
  },
  methods: {
    handleFileChange(e) {
      const fileList = Array.from(e.target.files);
      this.files = fileList.map(file => ({
        file,
        name: file.name,
        size: file.size,
        progress: 0,
        fileKey: generateFileKey(file),
        chunks: Math.ceil(file.size / (5 * 1024 * 1024)) // 5MB分片
      }));
    },
    async startUpload() {
      for (const fileInfo of this.files) {
        await this.uploadFileByChunk(fileInfo);
      }
    },
    async uploadFileByChunk(fileInfo) {
      const { file, chunks } = fileInfo;
      const chunkSize = 5 * 1024 * 1024; // 5MB每片
      
      for (let i = 0; i < chunks; i++) {
        const start = i * chunkSize;
        const end = Math.min(file.size, start + chunkSize);
        const chunk = file.slice(start, end);
        
        // 加密分片
        const encryptedChunk = await encryptFile(chunk, 'SM4');
        
        const formData = new FormData();
        formData.append('file', encryptedChunk);
        formData.append('chunkIndex', i);
        formData.append('chunks', chunks);
        formData.append('fileKey', fileInfo.fileKey);
        formData.append('fileName', file.name);
        formData.append('filePath', file.webkitRelativePath || '');
        
        try {
          await uploadFile(formData, progress => {
            fileInfo.progress = Math.floor(
              (i * 100 + progress * (100 / chunks)) / chunks
            );
          });
        } catch (error) {
          console.error('上传失败:', error);
          break;
        }
      }
    }
  }
};

后端实现方案

// Spring Boot控制器示例 - 文件分片上传
@RestController
@RequestMapping("/api/file")
public class FileUploadController {
    
    @Autowired
    private FileService fileService;
    
    @PostMapping("/upload")
    public ResponseEntity uploadFile(
            @RequestParam("file") MultipartFile file,
            @RequestParam("chunkIndex") int chunkIndex,
            @RequestParam("chunks") int chunks,
            @RequestParam("fileKey") String fileKey,
            @RequestParam(value = "filePath", required = false) String filePath,
            HttpSession session) {
        
        try {
            // 解密文件分片
            byte[] decryptedData = CryptoUtil.decrypt(file.getBytes(), "SM4");
            
            // 处理文件上传
            FileUploadResult result = fileService.handleFileUpload(
                fileKey,
                file.getOriginalFilename(),
                filePath,
                decryptedData,
                chunkIndex,
                chunks,
                session.getId()
            );
            
            return ResponseEntity.ok(result);
        } catch (Exception e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                .body(new ErrorResponse("文件上传失败: " + e.getMessage()));
        }
    }
    
    @GetMapping("/resumeInfo")
    public ResponseEntity getResumeInfo(
            @RequestParam("fileKey") String fileKey,
            HttpSession session) {
        try {
            ResumeInfo info = fileService.getResumeInfo(fileKey, session.getId());
            return ResponseEntity.ok(info);
        } catch (Exception e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                .body(new ErrorResponse("获取续传信息失败: " + e.getMessage()));
        }
    }
}

断点续传实现

// 断点续传服务实现
@Service
public class FileServiceImpl implements FileService {
    
    @Autowired
    private FileUploadRepository uploadRepository;
    
    @Autowired
    private StorageService storageService;
    
    @Override
    public FileUploadResult handleFileUpload(String fileKey, String fileName, 
            String filePath, byte[] data, int chunkIndex, 
            int totalChunks, String sessionId) {
        
        // 检查是否已有上传记录
        FileUploadRecord record = uploadRepository.findByFileKey(fileKey);
        
        if (record == null) {
            record = new FileUploadRecord();
            record.setFileKey(fileKey);
            record.setFileName(fileName);
            record.setFilePath(filePath);
            record.setTotalChunks(totalChunks);
            record.setSessionId(sessionId);
            record.setUploadedChunks(new ArrayList<>());
            uploadRepository.save(record);
        }
        
        // 存储分片
        String chunkKey = fileKey + "_" + chunkIndex;
        storageService.storeChunk(chunkKey, data);
        
        // 更新上传记录
        if (!record.getUploadedChunks().contains(chunkIndex)) {
            record.getUploadedChunks().add(chunkIndex);
            uploadRepository.save(record);
        }
        
        // 检查是否所有分片已上传完成
        if (record.getUploadedChunks().size() == totalChunks) {
            mergeFileChunks(record);
            return new FileUploadResult(true, 100, "文件上传完成");
        }
        
        int progress = (int) ((record.getUploadedChunks().size() * 100.0) / totalChunks);
        return new FileUploadResult(false, progress, "分片上传成功");
    }
    
    private void mergeFileChunks(FileUploadRecord record) {
        // 合并所有分片为完整文件
        List chunks = new ArrayList<>();
        for (int i = 0; i < record.getTotalChunks(); i++) {
            String chunkKey = record.getFileKey() + "_" + i;
            byte[] chunkData = storageService.getChunk(chunkKey);
            chunks.add(chunkData);
        }
        
        // 合并并存储最终文件
        byte[] fileData = mergeChunks(chunks);
        storageService.storeFinalFile(record.getFileKey(), fileData);
        
        // 清理临时分片
        for (int i = 0; i < record.getTotalChunks(); i++) {
            String chunkKey = record.getFileKey() + "_" + i;
            storageService.deleteChunk(chunkKey);
        }
        
        // 更新记录为已完成
        record.setCompleted(true);
        uploadRepository.save(record);
    }
}

文件夹结构保持实现

// 文件夹结构处理
public class FileStructureService {
    
    public void saveFolderStructure(String basePath, List fileItems) {
        Map> directoryMap = new HashMap<>();
        
        // 按目录分组
        for (FileItem item : fileItems) {
            String dirPath = extractDirectoryPath(item.getFilePath());
            directoryMap.computeIfAbsent(dirPath, k -> new ArrayList<>()).add(item);
        }
        
        // 保存目录结构到数据库
        for (Map.Entry> entry : directoryMap.entrySet()) {
            saveDirectoryRecord(basePath, entry.getKey(), entry.getValue());
        }
    }
    
    private String extractDirectoryPath(String filePath) {
        int lastSlash = filePath.lastIndexOf('/');
        return lastSlash > 0 ? filePath.substring(0, lastSlash) : "";
    }
    
    private void saveDirectoryRecord(String basePath, String dirPath, List files) {
        DirectoryRecord record = new DirectoryRecord();
        record.setBasePath(basePath);
        record.setPath(dirPath);
        record.setFileCount(files.size());
        
        // 计算目录大小
        long size = files.stream().mapToLong(FileItem::getSize).sum();
        record.setTotalSize(size);
        
        // 保存到数据库
        directoryRepository.save(record);
    }
}

非打包下载实现

// 文件夹下载服务
@Service
public class FolderDownloadService {
    
    @Autowired
    private DirectoryRepository directoryRepository;
    
    @Autowired
    private FileRecordRepository fileRecordRepository;
    
    @Autowired
    private StorageService storageService;
    
    public StreamingResponseBody downloadFolder(String folderId, HttpServletResponse response) {
        // 获取文件夹信息
        DirectoryRecord folder = directoryRepository.findById(folderId)
            .orElseThrow(() -> new ResourceNotFoundException("文件夹不存在"));
        
        // 设置响应头
        response.setContentType("application/octet-stream");
        response.setHeader("Content-Disposition", 
            "attachment; filename=\"" + folder.getName() + "\"");
        
        // 创建ZIP输出流(不实际打包，流式传输)
        return outputStream -> {
            List files = fileRecordRepository.findByDirectoryId(folderId);
            
            for (FileRecord file : files) {
                // 设置当前文件头
                response.setHeader("X-File-Name", file.getName());
                response.setHeader("X-File-Path", file.getPath());
                response.setHeader("X-File-Size", String.valueOf(file.getSize()));
                
                // 流式传输文件内容
                try (InputStream inputStream = storageService.getFileStream(file.getStorageKey())) {
                    byte[] buffer = new byte[1024 * 1024]; // 1MB buffer
                    int bytesRead;
                    while ((bytesRead = inputStream.read(buffer)) != -1) {
                        outputStream.write(buffer, 0, bytesRead);
                        outputStream.flush();
                    }
                }
                
                // 文件分隔标记
                outputStream.write("---FILE_SEPARATOR---".getBytes());
                outputStream.flush();
            }
        };
    }
}

兼容性处理方案

IE8兼容处理

// IE8兼容的上传方案
function setupIe8Uploader() {
  if (navigator.userAgent.indexOf('MSIE 8.0') > -1) {
    // IE8使用ActiveX或Flash方案
    var uploader = new ActiveXObject("YourUploader.ActiveX");
    
    uploader.onprogress = function(file, loaded, total) {
      updateProgress(file, (loaded / total) * 100);
    };
    
    uploader.oncomplete = function(file) {
      console.log('上传完成:', file);
    };
    
    // 注册事件处理
    document.getElementById('ie8UploadBtn').onclick = function() {
      var fileInput = document.getElementById('ie8FileInput');
      uploader.upload(fileInput.value);
    };
  }
}

多框架适配方案

// React适配器组件
import React from 'react';
import { useFileUpload } from './useFileUpload';

export const FileUploaderReact = ({ onUploadComplete }) => {
  const { files, handleFileChange, startUpload, progress } = useFileUpload();
  
  return (
    
      
      上传文件
      {files.map((file, index) => (
        
          {file.name}
          
        
      ))}
    
  );
};

// Vue3 Composition API
import { ref } from 'vue';
import { uploadFile } from '@/api/file';

export function useFileUpload() {
  const files = ref([]);
  
  const handleFileChange = (event) => {
    files.value = Array.from(event.target.files).map(file => ({
      file,
      name: file.name,
      progress: 0
    }));
  };
  
  const startUpload = async () => {
    for (const fileInfo of files.value) {
      await uploadFile(fileInfo.file, (progress) => {
        fileInfo.progress = progress;
      });
    }
  };
  
  return { files, handleFileChange, startUpload };
}

性能优化措施

分片上传/下载：5MB分片大小平衡网络传输和服务器负载
内存优化：流式处理避免大文件内存驻留
并发控制：限制同时上传/下载的连接数
断点信息存储：使用数据库持久化上传状态
缓存策略：热数据缓存减少数据库访问
负载均衡：多服务器分担传输负载

安全实施方案

加密处理核心代码

// 加密解密服务
@Service
public class CryptoService {
    
    private static final String AES_ALGORITHM = "AES/CBC/PKCS5Padding";
    private static final String SM4_ALGORITHM = "SM4/CBC/PKCS7Padding";
    
    public byte[] encrypt(byte[] data, String algorithm, String key) {
        try {
            Cipher cipher = Cipher.getInstance(algorithm);
            SecretKeySpec secretKey = new SecretKeySpec(key.getBytes(), 
                algorithm.startsWith("AES") ? "AES" : "SM4");
            
            // 使用固定IV保证可恢复性(实际项目应安全存储IV)
            byte[] iv = new byte[16];
            Arrays.fill(iv, (byte) 0x01);
            IvParameterSpec ivSpec = new IvParameterSpec(iv);
            
            cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec);
            return cipher.doFinal(data);
        } catch (Exception e) {
            throw new CryptoException("加密失败", e);
        }
    }
    
    public byte[] decrypt(byte[] encryptedData, String algorithm, String key) {
        try {
            Cipher cipher = Cipher.getInstance(algorithm);
            SecretKeySpec secretKey = new SecretKeySpec(key.getBytes(), 
                algorithm.startsWith("AES") ? "AES" : "SM4");
            
            byte[] iv = new byte[16];
            Arrays.fill(iv, (byte) 0x01);
            IvParameterSpec ivSpec = new IvParameterSpec(iv);
            
            cipher.init(Cipher.DECRYPT_MODE, secretKey, ivSpec);
            return cipher.doFinal(encryptedData);
        } catch (Exception e) {
            throw new CryptoException("解密失败", e);
        }
    }
    
    public String generateFileKey(File file) {
        // 生成唯一文件标识(结合文件属性和时间戳)
        String rawKey = file.getName() + file.length() + System.currentTimeMillis();
        return DigestUtils.md5DigestAsHex(rawKey.getBytes());
    }
}

部署架构建议

高可用部署方案

[客户端] → [CDN/负载均衡] → [Web服务器集群] 
                              ↓
                      [应用服务器集群] → [Redis集群(会话/缓存)]
                              ↓
                      [数据库集群(主从)]   [文件存储集群]
                                         (OSS/本地存储)

私有云部署配置

# application-prd.yml
storage:
  type: oss # 可选 local, oss, s3
  oss:
    endpoint: https://your-oss-endpoint.aliyuncs.com
    accessKeyId: your-access-key
    accessKeySecret: your-secret-key
    bucketName: your-bucket-name
  local:
    rootPath: /data/file-storage
    
server:
  maxFileSize: 100GB
  maxRequestSize: 101GB
  chunkSize: 5MB

security:
  crypto:
    defaultAlgorithm: SM4
    aesKey: your-aes-key
    sm4Key: your-sm4-key