VUE3大文件分片+worker优化分片速度+node.js示例

已于 2025-05-06 10:39:59 修改
阅读量180 收藏
点赞数 6
文章标签： node.js
于 2025-05-06 10:14:52 首次发布
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/zww2446/article/details/147728477
版权

首先是文件上传以及分片，话不多说，直接贴代码

<template>
  <div>
    <input type="file" @change="handleFileChange" class="file" />
  </div>
</template>

<script setup>

import { ref } from 'vue'
const CHUNK_SIZE = 2 * 1024 * 1024 // 每片 2MB
const THREAD_COUNT = navigator.hardwareConcurrency || 4 // 线程数
import SparkMD5 from 'spark-md5'
import axios from 'axios';
const CONCURRENCY = 5; // 并发数

async function handleFileChange (e) {
  console.log('文件变化了', e);

  const file = e.target.files[0]
  const _fileName = file.name
  console.time('开始')
  const chunks = await cutFile(file)
  console.timeEnd('开始')
  console.log('分片:', chunks)
}

async function cutFile (file) {
  const fileId = await calculateFileMd5(file); // ⬅️ 先计算整个文件的 MD5
  return new Promise((resolve, reject) => {
    const chunkCount = Math.ceil(file.size / CHUNK_SIZE); // 计算分片数量

    const threadChunkCount = Math.ceil(chunkCount / THREAD_COUNT); // 计算每个线程分片数量
    const result = [];
    let finisCount = 0; // 计算完成数量
    for (let i = 0; i < THREAD_COUNT; i++) {
      const worker = new Worker(new URL('./文件分片/线程/worker.js', import.meta.url), {
        type: 'module'
      });
      let start = i * threadChunkCount; // 开始位置
      let end = Math.min(start + threadChunkCount, chunkCount); // 结束位置
      worker.postMessage({
        file, // 切割文件
        start, // 开始位置
        end,// 结束位置
        CHUNK_SIZE, // 每片大小
      });
      worker.onmessage = (e) => {
        worker.terminate();
        result[i] = e.data; // 获取worker返回的数据
        finisCount++;

        if (finisCount === THREAD_COUNT) {
          resolve({ result: result.flat(), fileId });
        }
      }
    }
  })

}

function calculateFileMd5 (file, chunkSize = 2 * 1024 * 1024) {
  return new Promise((resolve, reject) => {
    const spark = new SparkMD5.ArrayBuffer();
    const chunks = Math.ceil(file.size / chunkSize);
    let currentChunk = 0;
    const reader = new FileReader();

    reader.onload = (e) => {
      spark.append(e.target.result);
      currentChunk++;
      if (currentChunk < chunks) {
        loadNext();
      } else {
        resolve(spark.end());
      }
    };
    reader.onerror = reject;
    function loadNext () {
      const start = currentChunk * chunkSize;
      const end = Math.min(start + chunkSize, file.size);
      const blob = file.slice(start, end);
      reader.readAsArrayBuffer(blob);
    }
    loadNext();
  });
}

</script>

<style lang="scss" scoped>
</style>

worker.js中的代码

import { createChunk } from './c'; // 保留原来的 createChunk（每个 chunk 的 MD5 或数据）

onmessage = async function (e) {
    const { file, start, end, CHUNK_SIZE, fileId } = e.data;
    const result = [];

    for (let i = start; i < end; i++) {

        const chunk = await createChunk(file, i, CHUNK_SIZE);
        result.push({ ...chunk }); // ⬅️ 添加 fileId
    }

    postMessage(result); // ⬅️ 添加 fileId
};

createChunk函数

import SparkMD5 from 'spark-md5'

export function createChunk (file, index, chunkSize) {

    return new Promise((resolve, reject) => {
        const start = index * chunkSize; // 开始位置
        const end = start + chunkSize; // 结束位置
        const spark = new SparkMD5.ArrayBuffer(); // 创建一个md5对象
        const fileReader = new FileReader(); // 创建一个文件读取器
        const blob = file.slice(start, end); // 切割文件
        fileReader.onload = (e) => { // 文件读取完成
            spark.append(e.target.result);
            const checkObj = {
                start,
                end,
                index,
                hash: spark.end(), // 计算md5值
                blob
            }

            resolve(checkObj)
        }
        fileReader.readAsArrayBuffer(blob);
    })
}

解释一下，checkUploadedChunks这个方法主要是给整个文件生成一个MD5编号。cutFile这个方法是给每个分片生成一个MD5编号，主要是为了上传使用

接下来上node（文件名：server.js）代码，重要：启动命令：node server.js

const express = require('express');
const multer = require('multer');
const fs = require('fs');
const fsp = require('fs/promises');
const path = require('path');
const cors = require('cors');

const app = express();
const PORT = 3000;

app.use(cors());
app.use(express.json());

// 配置上传目录
const UPLOAD_DIR = path.resolve(__dirname, 'uploads');
const MERGE_DIR = path.resolve(__dirname, 'merged');

// 动态创建上传目录
function getChunkDir (fileId) {
  return path.resolve(UPLOAD_DIR, fileId);
}

const storage = multer.diskStorage({
  destination: function (req, file, cb) {
    const { fileId } = req.body;
    const chunkDir = getChunkDir(fileId);
    fs.mkdirSync(chunkDir, { recursive: true });
    cb(null, chunkDir);
  },
  filename: function (req, file, cb) {
    const { index } = req.body;
    cb(null, `chunk-${index}`);
  }
});

const upload = multer({ storage });

// 上传接口
app.post('/api/upload', upload.single('chunk'), (req, res) => {
  res.send({ message: '分片上传成功' });
});

// 检查已上传分片
app.post('/api/check', async (req, res) => {
  const { fileId } = req.body;
  const chunkDir = getChunkDir(fileId);
  if (!fs.existsSync(chunkDir)) {
    return res.send({ uploaded: [] });
  }
  const files = await fsp.readdir(chunkDir);
  const uploaded = files.map(name => name.split('-')[1]); // 返回 chunk id（例如 index）
  res.send({ uploaded });
});

// 合并分片
app.post('/api/merge', async (req, res) => {
  const { fileId, fileName } = req.body;
  const chunkDir = getChunkDir(fileId);
  const filePath = path.resolve(MERGE_DIR, fileName);
  await fsp.mkdir(MERGE_DIR, { recursive: true });

  const chunkFiles = await fsp.readdir(chunkDir);
  const sortedChunks = chunkFiles
    .sort((a, b) => Number(a.split('-')[1]) - Number(b.split('-')[1]));

  const writeStream = fs.createWriteStream(filePath);

  for (const chunkFile of sortedChunks) {
    const chunkPath = path.resolve(chunkDir, chunkFile);
    const data = fs.readFileSync(chunkPath);
    writeStream.write(data);
  }

  writeStream.end();
  writeStream.on('finish', () => {
    res.send({ message: '文件合并成功' });
    fs.rmSync(chunkDir, { recursive: true, force: true }); // 删除临时分片
  });
});

app.listen(PORT, () => {
  console.log(`🚀 Server running at http://localhost:${PORT}`);
});

完整代码示例

<template>
  <div>
    <input type="file" @change="handleFileChange" class="file" />
  </div>
</template>

<script setup>

import { ref } from 'vue'
const CHUNK_SIZE = 2 * 1024 * 1024 // 每片 2MB
const THREAD_COUNT = navigator.hardwareConcurrency || 4 // 线程数
import SparkMD5 from 'spark-md5'
import axios from 'axios';
const CONCURRENCY = 5; // 并发数

async function handleFileChange (e) {
  console.log('文件变化了', e);

  const file = e.target.files[0]
  const _fileName = file.name
  console.time('开始')
  const chunks = await cutFile(file)
  console.timeEnd('开始')
  console.log('分片:', chunks)

  const uploadedList = await checkUploadedChunks(chunks.fileId);
  console.log('已上传分片:', uploadedList);
  const toUploadChunks = chunks.result.filter(c => !uploadedList.includes(c.hash));
  console.log('待上传分片:', toUploadChunks);
  await uploadChunks(chunks.fileId, toUploadChunks); //
  await mergeFile(chunks.fileId, _fileName);
}

async function checkUploadedChunks (fileId) {
  const res = await axios.post('http://localhost:3000/api/check', { fileId });
  return res.data.uploaded || [];
}
async function uploadChunks (fileId, chunks) {
  let completed = 0;

  if (!fileId || !chunks.length) {
    alert('没有需要上传的分片！');
    return;
  }

  async function uploadChunk (chunk) {
    const form = new FormData();
    form.append('fileId', fileId);
    form.append('chunkId', chunk.hash);
    form.append('index', chunk.index);
    form.append('chunk', chunk.blob);

    await axios.post('http://localhost:3000/api/upload', form);
    completed++;
    let check = ((completed / chunks.length) * 100).toFixed(2); // 可选
    console.log(check, "%");

  }

  let index = 0;
  const pool = new Array(CONCURRENCY).fill(null).map(() => {
    return new Promise(async function next (resolve) {
      if (index >= chunks.length) return resolve();

      const chunk = chunks[index++];
      await uploadChunk(chunk);
      next(resolve);
    });
  });

  await Promise.all(pool);
}


async function mergeFile (fileId, fileName) {
  await axios.post('http://localhost:3000/api/merge', { fileId, fileName });
}



async function cutFile (file) {
  const fileId = await calculateFileMd5(file); // ⬅️ 先计算整个文件的 MD5
  return new Promise((resolve, reject) => {
    const chunkCount = Math.ceil(file.size / CHUNK_SIZE); // 计算分片数量

    const threadChunkCount = Math.ceil(chunkCount / THREAD_COUNT); // 计算每个线程分片数量
    const result = [];
    let finisCount = 0; // 计算完成数量
    for (let i = 0; i < THREAD_COUNT; i++) {
      const worker = new Worker(new URL('./文件分片/线程/worker.js', import.meta.url), {
        type: 'module'
      });
      let start = i * threadChunkCount; // 开始位置
      let end = Math.min(start + threadChunkCount, chunkCount); // 结束位置
      worker.postMessage({
        file, // 切割文件
        start, // 开始位置
        end,// 结束位置
        CHUNK_SIZE, // 每片大小
      });
      worker.onmessage = (e) => {
        worker.terminate();
        result[i] = e.data; // 获取worker返回的数据
        finisCount++;

        if (finisCount === THREAD_COUNT) {
          resolve({ result: result.flat(), fileId });
        }
      }
    }
  })

}

function calculateFileMd5 (file, chunkSize = 2 * 1024 * 1024) {
  return new Promise((resolve, reject) => {
    const spark = new SparkMD5.ArrayBuffer();
    const chunks = Math.ceil(file.size / chunkSize);
    let currentChunk = 0;
    const reader = new FileReader();

    reader.onload = (e) => {
      spark.append(e.target.result);
      currentChunk++;
      if (currentChunk < chunks) {
        loadNext();
      } else {
        resolve(spark.end());
      }
    };
    reader.onerror = reject;
    function loadNext () {
      const start = currentChunk * chunkSize;
      const end = Math.min(start + chunkSize, file.size);
      const blob = file.slice(start, end);
      reader.readAsArrayBuffer(blob);
    }
    loadNext();
  });
}


// function calculateFileMd5 (file, chunkSize = CHUNK_SIZE) {
//   return new Promise((resolve, reject) => {
//     const worker = new Worker(new URL('./文件分片/线程/md5.worker.js', import.meta.url), {
//       type: 'module',
//     });

//     worker.postMessage({ file, chunkSize });

//     worker.onmessage = (e) => {
//       const { fileId, error } = e.data;
//       worker.terminate();
//       if (error) {
//         reject(error);
//       } else {
//         resolve(fileId);
//       }
//     };

//     worker.onerror = (err) => {
//       worker.terminate();
//       reject(err);
//     };
//   });
// }





</script>

<style lang="scss" scoped>
</style>

重点：这个思路就是将整个的文件ID传给后端，如果没有上传，就返回给你空数组，上传了，则返回上传的id,然后根据id筛选出没有上传的数据，进行接口上传，其中也写了并发数以及线程数量，速度优化了50%

接下来把断点续传，以及样式和分片进度、上传进度、速度优化交给你们来写

上上签6155
关注
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值