大文件分片上传、断点续传（附带前后端demo）

序

大文件上传，断点续传，秒传，作为高频考察的技术点，大多数人都是知其然而不知其所以然，下面我们从前后端一起的角度来探究一番，
相信只要你肯花一点时间认真地理解它，你就会发现你花了一些时间。。。

思路图

前端核心代码片段

按照指定size进行文件切片

    createChunk(file, size = 512 * 1024) {
      const chunkList = [];
      let cur = 0;
      while (cur < file.size) {
        // 使用slice方法切片
        chunkList.push({ file: file.slice(cur, cur + size) });
        cur += size;
      }
      return chunkList;
    }

文件使用js-md5转哈希

    handleFileChange(e) {
      let fileReader = new FileReader();
      fileReader.readAsDataURL(e.target.files[0]);
      fileReader.onload = function (e2) {
          const hexHash = md5(e2.target.result)+'.'+that.fileObj.file.name.split('.').pop();
      };
    }

切片转成表单对象后，配置成一个请求列表。

const requestList = noUploadChunks.map(({ file, fileName, index, chunkName }) => {
          const formData = new FormData();
          formData.append("file", file);
          formData.append("fileName", fileName);
          formData.append("chunkName", chunkName);
          return { formData, index };
        })
        .map(({ formData, index }) =>
          axiosRequest({
            url: "http://localhost:3000/upload",
            data: formData
          })
        )

前端完整代码

<template>
  <div>
    <input type="file" @change="handleFileChange" />
    <el-button @click="uploadChunks"> 上传 </el-button>
    <!-- <el-button @click="pauseUpload"> 暂停 </el-button> -->
    <div style="width: 300px">
          总进度：
          <el-progress :percentage="tempPercent"></el-progress>
          切片进度：
          <div v-for="(item,index) in fileObj.chunkList" :key="index">
            <span>{{ item.chunkName }}：</span>
            <el-progress :percentage="item.percent"></el-progress>
          </div>
    </div>
  </div>
</template>

<script>
import axios from "axios";
import md5 from 'js-md5'
 const CancelToken = axios.CancelToken;
 let source = CancelToken.source();

function axiosRequest({
  url,
  method = "post",
  data,
  headers = {},
  onUploadProgress = (e) => e, // 进度回调
}) {
  return new Promise((resolve, reject) => {
    axios[method](url, data, {
      headers,
      onUploadProgress, // 传入监听进度回调
      cancelToken: source.token
    })
      .then((res) => {
        resolve(res);
      })
      .catch((err) => {
        reject(err);
      });
  });
}
export default {
  data() {
    return {
      fileObj: {
        file: null,
        chunkList:[]
      },
      tempPercent:0
    };
  },
  methods: {
    handleFileChange(e) {
      const [file] = e.target.files;
      if (!file) return;
      this.fileObj.file = file;
      const fileObj = this.fileObj;
      if (!fileObj.file) return;
      const chunkList = this.createChunk(fileObj.file);
      console.log(chunkList); // 看看chunkList长什么样子
      let that = this
      // 获取此视频的哈希值,作为名字
      let fileReader = new FileReader();
      fileReader.readAsDataURL(e.target.files[0]);
      fileReader.onload = function (e2) {
          const hexHash = md5(e2.target.result)+'.'+that.fileObj.file.name.split('.').pop();
          that.fileObj.name = hexHash
          that.fileObj.chunkList = chunkList.map(({ file }, index) => ({
          file,
          size: file.size,
          percent: 0,
          chunkName: `${hexHash}-${index}`,
          fileName: hexHash,
          index,
        }))
      };
    },
    createChunk(file, size = 512 * 1024) {
      const chunkList = [];
      let cur = 0;
      while (cur < file.size) {
        // 使用slice方法切片
        chunkList.push({ file: file.slice(cur, cur + size) });
        cur += size;
      }
      return chunkList;
    },
    async uploadChunks() {
      const {uploadedList,shouldUpload} = await this.verifyUpload()
      // 如果存在这个文件
      if(!shouldUpload){
        console.log('秒传成功')
        this.fileObj.chunkList.forEach(item => {
          item.percent = 100
        });
        return 
      }
      let noUploadChunks = []
      if(uploadedList&&uploadedList.length>0 &&uploadedList.length !== this.fileObj.chunkList.length){
        // 如果存在切片。只上传没有的切片
        noUploadChunks =  this.fileObj.chunkList.filter(item=>{
          if(uploadedList.includes(item.chunkName)){
            item.percent = 100
          }
          return !uploadedList.includes(item.chunkName)
        })
      }else{
        noUploadChunks = this.fileObj.chunkList
      }
      const requestList = noUploadChunks.map(({ file, fileName, index, chunkName }) => {
          const formData = new FormData();
          formData.append("file", file);
          formData.append("fileName", fileName);
          formData.append("chunkName", chunkName);
          return { formData, index };
        })
        .map(({ formData, index }) =>
          axiosRequest({
            url: "http://localhost:3000/upload",
            data: formData,
            onUploadProgress: this.createProgressHandler(
              this.fileObj.chunkList[index]
            ), // 传入监听上传进度回调
          })
        );
      await Promise.all(requestList); // 使用Promise.all进行请求
      this.mergeChunks()
    },
    createProgressHandler(item) {
      return (e) => {
        // 设置每一个切片的进度百分比
        item.percent = parseInt(String((e.loaded / e.total) * 100));
      };
    },
    mergeChunks(size = 512 * 1024) {
      axiosRequest({
         url: "http://localhost:3000/merge",
         headers: {
           "content-type": "application/json",
         },
         data: JSON.stringify({ 
          size,
           fileName: this.fileObj.name
         }),
       });
     },
     pauseUpload() {
      source.cancel("中断上传!");
       source = CancelToken.source(); // 重置source，确保能续传
     },
     async verifyUpload () {
       const { data } = await axiosRequest({
         url: "http://localhost:3000/verify",
         headers: {
           "content-type": "application/json",
         },
         data: JSON.stringify({
           fileName:this.fileObj.name
        }),
      });
      return data
     }
  },
  computed: {
    totalPercent() {
      const fileObj = this.fileObj;
      if (fileObj.chunkList.length === 0) return 0;
      const loaded = fileObj.chunkList
        .map(({ size, percent }) => size * percent)
       .reduce((pre, next) => pre + next);
      return parseInt((loaded / fileObj.file.size).toFixed(2));
    },
  },
  watch: {
      totalPercent (newVal) {
           if (newVal > this.tempPercent) this.tempPercent = newVal
       }
   },
};
</script>

<style lang="scss" scoped></style>

服务端接收代码（用node.js模拟）

const http = require("http");
const path = require("path");
const fse = require("fs-extra");
const multiparty = require("multiparty");

const server = http.createServer();
const UPLOAD_DIR = path.resolve(__dirname, ".", `qiepian`); // 切片存储目录

server.on("request", async (req, res) => {
  res.setHeader("Access-Control-Allow-Origin", "*");
  res.setHeader("Access-Control-Allow-Headers", "*");
  res.setHeader('Content-Type','text/html; charset=utf-8');
  if (req.method === "OPTIONS") {
    res.status = 200;
    res.end();
    return;
  }
  console.log(req.url);

  if (req.url === "/upload") {
    const multipart = new multiparty.Form();

    multipart.parse(req, async (err, fields, files) => {
      if (err) {
        console.log("errrrr", err);
        return;
      }
      const [file] = files.file;
      const [fileName] = fields.fileName;
      const [chunkName] = fields.chunkName;
      // 保存切片的文件夹的路径，比如  张远-嘉宾.flac-chunks
      const chunkDir = path.resolve(UPLOAD_DIR, `${fileName}-chunks`);
      // // 切片目录不存在，创建切片目录
      if (!fse.existsSync(chunkDir)) {
        await fse.mkdirs(chunkDir);
      }
      // 把切片移动到切片文件夹
      await fse.move(file.path, `${chunkDir}/${chunkName}`);
      res.end(
        JSON.stringify({
          code: 0,
          message: "切片上传成功",
        })
      );
    });
  }

  // 合并切片
  // 接收请求的参数
  const resolvePost = (req) =>
    new Promise((res) => {
      let chunk = "";
      req.on("data", (data) => {
        chunk += data;
      });
      req.on("end", () => {
        res(JSON.parse(chunk));
      });
    });
  const pipeStream = (path, writeStream) => {
    console.log("path", path);
    return new Promise((resolve) => {
      const readStream = fse.createReadStream(path);
      readStream.on("end", () => {
        // fse.unlinkSync(path); // 删除切片文件
        resolve();
      });
      readStream.pipe(writeStream);
    });
  };

  // 合并切片
  const mergeFileChunk = async (filePath, fileName, size) => {
    // filePath：你将切片合并到哪里，的路径
    const chunkDir = path.resolve(UPLOAD_DIR, `${fileName}-chunks`);
    let chunkPaths = null;
    // 获取切片文件夹里所有切片，返回一个数组
    chunkPaths = await fse.readdir(chunkDir);
    // 根据切片下标进行排序
    // 否则直接读取目录的获得的顺序可能会错乱
    chunkPaths.sort((a, b) => a.split("-")[1] - b.split("-")[1]);
    const arr = chunkPaths.map((chunkPath, index) => {
      return pipeStream(
        path.resolve(chunkDir, chunkPath),
        // 指定位置创建可写流
        fse.createWriteStream(filePath, {
          start: index * size,
          end: (index + 1) * size,
        })
      );
    });
    await Promise.all(arr);
  };
  if (req.url === "/merge") {
    const data = await resolvePost(req);
    const { fileName, size } = data;
    const filePath = path.resolve(UPLOAD_DIR, fileName);
    await mergeFileChunk(filePath, fileName, size);
    res.end(
      JSON.stringify({
        code: 0,
        message: "文件合并成功",
      })
    );
  }
  if (req.url === "/verify") {
      // 返回已经上传切片名列表
      const createUploadedList = async fileName =>
              fse.existsSync(path.resolve(UPLOAD_DIR, fileName))
                  ? await fse.readdir(path.resolve(UPLOAD_DIR, fileName))
                  : [];
      const data = await resolvePost(req);
      const { fileName } = data;
      const filePath = path.resolve(UPLOAD_DIR, fileName);
      console.log(filePath)
      if (fse.existsSync(filePath)) {
          res.end(
              JSON.stringify({
                  shouldUpload: false
              })
          );
      } else {
          res.end(
              JSON.stringify({
                  shouldUpload: true,
                      uploadedList: await createUploadedList(`${fileName}-chunks`)
              })
          );
      }
  }
});

server.listen(3000, () => console.log("正在监听 3000 端口"));

demo测试

• npm安装好缺的依赖，
• 运行前端代码。
• node xxx.js 运行后端代码。