上传大文件 —— 切片上传

最新推荐文章于 2024-06-04 17:05:25 发布

裘乡

最新推荐文章于 2024-06-04 17:05:25 发布

阅读量780

点赞数 11

文章标签： javascript 前端 html

本文链接：https://blog.csdn.net/QiuXiang_/article/details/139005449

版权

切片上传是什么？

将大文件分割成小块（分片），分别上传到服务器，然后在服务器端重新组装成完整的文件。这种方法可以减少单次上传的负载，同时更容易处理网络中断或失败的情况。

如何使用

首先明确切片上传的四个步骤

前端调用接口，查看之前是否上传过当前文件的片段，用来得出是要上传当前文件的全部片段还是部分片段
前端将文件进行切片，并根据实际要上传的片段来请求接口
若请求都顺利完成，前端调用后端的合并切片的接口
后端进行校验后，进行大文件的合并

前端切片及上传

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>File Upload</title>
</head>
<body>
<input type="file" id="fileInput">
<button id="uploadButton">Upload</button>

<script>
  /**
   * 上传单个文件
   * @param chunk 要上传的文件片段
   * @param index 上传片段的索引下标
   * @param fileName 上传文件名称
   * @returns {Promise<void>}
   */
  const uploadChunk = async (chunk, index, fileName) => {
    const formData = new FormData();
    formData.append('file', chunk, `${fileName}.part${index}`);
    await fetch('http://localhost:3000/upload', {
      method: 'POST',
      body: formData
    });
  };

  /**
   * 获取已经上传的片段
   * @param fileName
   * @returns {Promise<*>}
   */
  const getUploadedChunks = async (fileName) => {
    const response = await fetch(`http://localhost:3000/uploadedChunks?fileName=${fileName}`);
    const data = await response.json();
    return data.uploadedChunks;
  };

  // 点击上传
  document.getElementById('uploadButton').addEventListener('click', async () => {
    const fileInput = document.getElementById('fileInput');
    const file = fileInput.files[0];
    const chunkSize = 1024 * 1024 * 50; // 50MB一片段
    const chunkCount = Math.ceil(file.size / chunkSize);
    
    // 获取已上传的片段，来确认要上传的片段
    const uploadedChunks = await getUploadedChunks(file.name);
    const uploadedChunksSet = new Set(uploadedChunks);
    const uploadList = [];
    for (let i = 0; i < chunkCount; i++) {
      if (uploadedChunksSet.has(`${file.name}.part${i}`)) {
        console.log(`Chunk ${i} already uploaded, skipping...`);
        continue;
      }
      const start = i * chunkSize;
      const end = Math.min(start + chunkSize, file.size);
      const chunk = file.slice(start, end);
      uploadList.push(uploadChunk(chunk, i, file.name));
    }

    // 若是所有上传都成功了，就会执行合并切片操作，不然则提示重新上传
    Promise.all(uploadList).then(() => {
      // 请求服务器合并切片
      fetch('http://localhost:3000/merge', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ fileName: file.name, chunkCount })
      })
        .then(response => response.json())
        .then(data => {
          console.log('File merged successfully:', data);
        })
        .catch(error => {
          console.error('Error:', error);
        });
    }).catch(error => {
      console.log('Upload failed, please upload again')
    });
  });
</script>
</body>
</html>

后端接收文件及合并切片

npm install express multer cors

const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const cors = require('cors');

const app = express();
app.use(cors());
app.use(express.json());
app.use(express.urlencoded({ extended: true }));

// 进行存储
const storage = multer.diskStorage({
  destination: (req, file, cb) => {
    const uploadPath = 'uploads/';
    if (!fs.existsSync(uploadPath)) {
      fs.mkdirSync(uploadPath);
    }
    cb(null, uploadPath);
  },
  filename: (req, file, cb) => {
    cb(null, file.originalname);
  }
});

const upload = multer({ storage: storage });

// 上传文件接口
app.post('/upload', upload.single('file'), (req, res) => {
  res.send({ message: 'Chunk uploaded successfully.' });
});

// 查询已经上传的文件切片
app.get('/uploadedChunks', (req, res) => {
  const { fileName } = req.query;
  const uploadPath = path.join(__dirname, 'uploads');
  const files = fs.readdirSync(uploadPath);
  const uploadedChunks = files.filter(file => file.startsWith(fileName));
  res.send({ uploadedChunks });
});

// 对上传的文件进行合并
app.post('/merge', async (req, res) => {
  const { fileName, chunkCount } = req.body;
  const filePath = path.join(__dirname, 'merged', fileName);
  const writeStream = fs.createWriteStream(filePath);

  try {
    // 判断是否是完整的文件切片
    for (let i = 0; i < chunkCount; i++) {
      const chunkPath = path.join(__dirname, 'uploads', `${fileName}.part${i}`);
      if (!fs.existsSync(chunkPath)) {
        throw new Error(`Chunk ${i} is missing`);
      }
      const data = fs.readFileSync(chunkPath);
      writeStream.write(data);
      fs.unlinkSync(chunkPath);
    }
    
    writeStream.end();
    writeStream.on('finish', () => {
      res.send({ message: 'File merged successfully.', filePath });
    });
  } catch (error) {
    console.error(error);
    res.status(500).send({ message: 'Failed to merge file.', error: error.message });
  }
});

const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
  console.log(`Server is running on port ${PORT}.`);
});