大文件分片上传

项目场景：

项目场景：我们有个机场和无人机的列表需要对,对应的固件版本做升级:
1.本地下载机场/无人机最新离线固件包
2.系统内上传机场/无人机最新离线固件包至服务器
3.在系统中对比当前版本与最新上传版本号
这就产生了往服务器上传固件包的情况,文件可能几百M,就用到了大文件上传

---

分片上传原理

1.通过file.slice将大文件chunks切成许多个大小相等的chunk,将每个chunk上传到服务器,服务端接收到许多个chunk后，合并为chunks
2.处理切片后的文件，给后端传一个;利用md5生成的每个文件的identifier，用来读取文件
3.然后就依次初始这些分片任务并依次上传,最后服务根据identifier,合并分片

分用到的插件

npm install --save spark-md5

 npm install --save promise-queue-plus

具体实现代码：

---

主要上传代码

<template>
  <div class="wrapForm">
    <el-form
      :model="modelForm"
      :rules="rules"
      ref="modelForm"
      label-width="100px"
      class="demo-ruleForm"
    >
      <el-form-item
        label="设备型号"
        prop="device_name"
      >
        <el-select
          clearable
          ref="dutyRef"
          filterable
          v-model="modelForm.device_name"
          placeholder="请选择"
          style="width:305px"
        >
          <el-option
            v-for="(item, index) in deviceModelDict"
            :key="index"
            :label="item.itemText"
            :value="item.itemValue"
          />
        </el-select>
      </el-form-item>
      <el-form-item
        label="文件上传"
        prop="fileUpload"
      >
        <el-upload
          class="upload-demo"
          ref="uploadRef"
          :limit="1"
          :on-error="handleError"
          :on-progress="handleProgress"
          :on-success="handleAvatarSuccess"
          action="/"
          multiple
          :http-request="customHttpRequest"
          :on-exceed="handleExceed"
        >
          <div class="uploadContent">
            <div class="uploadInp">
              <i class="el-icon-upload"></i>
              <div class="el-upload__text">上传文件</div>
            </div>
            <div
              class="el-upload__tip"
              slot="tip"
            >只支持格式:zip</div>
          </div>
        </el-upload>
        <div class="fileTips" v-if="fileParsingIs" >
          文件解析中.....
        </div>
        <div class="actionIcon" v-show="closeFlag">
          <i   class="el-icon-close" @click="clearFileFun"></i>
        </div>
        <div class="actionIcon" v-show="isSuccessFlag">
          <i v-if="!isHovering" @mouseover="isHovering = true" @mouseout="isHovering = false" class="el-icon-upload-success el-icon-circle-check" style="color: #69C04E;"></i>
          <i  v-else @mouseover="isHovering = false" @mouseout="isHovering = false" class="el-icon-close" @click="clearFileFun"></i>
        </div>
      </el-form-item>
    </el-form>
    <div class="btnWarp">
      <el-button @click="cancelFun('modelForm')">取消</el-button>
      <el-button
        :disabled="disabledUplaod"
        type="primary"
        @click="confimFirmware('modelForm')"
      >确&nbsp;&nbsp;定</el-button>
    </div>
  </div>
</template>
<script>
import md5 from "@/util/md5.js"; //计算文件的md5
import axios from 'axios'
import Queue from 'promise-queue-plus';
import JSZip from 'jszip'
import {
    getTaskInfoUpload,
    initTaskUpload,
    preSignUrl,
    preMergeUpload,
    saveFirmware,
    fileExists
  } from "@frame/api/admin/airManange.js";
export default {
    props:{
        firmwareList:{
          type: Array,
          default: [],
        },
        deviceModelDict:{
          type: Array,
          default: [],
        }
    },
    data() {
        return {
            isSuccessFlag:false,
            closeFlag:false,
            isHovering: false,
            fileParsingIs:false,
            fileUploadChunkQueue: {},
            percentageD:0,//上传进度
            disabledUplaod:true,
            // deviceNameList:[],
            firmwareUpdate:{
              version:'',
              time:''
            },
            modelForm: {
                device_name: "",
                fileUpload:''
            },
            rules: {
                device_name: [
                    { required: true, message: '请选择', trigger: 'change' },
                ],
                fileUpload: [
                    { required: true, message: '请选择', trigger: 'change' },
                ],
            },
            initDataMes:{},//上传文件信息
        };
    },
    watch:{
      percentageD(newVal){
        if (newVal&&newVal>0) {
          this.fileParsingIs = false;
        }
      },
    },
    computed:{
    },
    created() {
    },
    mounted() {
    },
    methods: { 
      // 取消
      cancelFun(formName){
        this.$emit("closeUploadDialog")
        this.$refs[formName].resetFields();
        this.clearForm()
      },
      // 清空from
      clearForm(){
        this.$refs.uploadRef.clearFiles();
        this.modelForm.device_name = ''
        this.isSuccessFlag = false
      },
      // 保存
      confimFirmware(formName){
         this.$refs[formName].validate((valid) => {
           if (valid) {
             if (!this.firmwareUpdate.version) {
               this.$message.error('固件版本号不能为空，请重新选择文件')
               return
             }
             if (!this.firmwareUpdate.time) {
               this.$message.error('固件更新日期不能为空，请重新选择文件')
               return
             }
            let data={
                deviceName:this.modelForm.device_name,//设备型号
                fileMd5:this.initDataMes.taskRecord.fileIdentifier,//文件md5码
                fileName:this.initDataMes.taskRecord.fileName,//文件名
                fileSize:this.initDataMes.taskRecord.totalSize,//文件大小
                fileUrl:this.initDataMes.path,//文件url
                productVersion:this.firmwareUpdate.version,//版本号
                releasedTime:this.firmwareUpdate.time,//版本更新日期,yyyyMMdd
            }
            saveFirmware(data).then(res=>{
              if (res.code == 0) {
                this.$emit("closeUploadDialog")
                this.$refs[formName].resetFields();
                this.clearForm()
                // 刷新父组件固件列表接口
                this.$parent.$parent.firmwarePageList()
              }
            })
          }
        });
      },
      /**
       * el-upload 自定义上传方法入口
       */
      async customHttpRequest(options) {
          const file = options.file;
          const fileSuffix = file.name.substring(file.name.lastIndexOf(".") + 1);
          const whiteList = ["zip"];
          if (whiteList.indexOf(fileSuffix) === -1) {
              this.$message.error("请上传zip格式文件");
              this.handleError(file)
              return false;
          }
          const zip = new JSZip();
          const contents = await zip.loadAsync(file);
          // 列出ZIP包中的所有文件
          for (let filename in contents.files) {
            let eleFile = filename.split('.');
            if (eleFile[eleFile.length-2] == 'cfg') {
              let itemFirm = filename.split('_');
              this.firmwareUpdate.version = itemFirm[itemFirm.length-2]
              this.firmwareUpdate.version = this.firmwareUpdate.version.replace(new RegExp('v', 'g'), '')
              this.firmwareUpdate.time = itemFirm[itemFirm.length-1].split('.')[0]
            }
          }
          const result = await fileExists({ deviceName: this.modelForm.device_name, version: this.firmwareUpdate.version });
          // 判断文件是否存在
          if (result.data == true) {
            this.disabledUplaod = true;
            this.$message.warning('固件版本包已存在，请重新选择')
            this.handleError(file)
            return
          }
          this.fileParsingIs = true
          this.closeFlag = true
          const task = await this.getTaskInfo(file); //首先匹配文件的md5，查询该md5是否存在，存在则可以直接添加
          this.initDataMes = task;
          if (task) {
              const { finished, path, taskRecord } = task
              const { fileIdentifier: identifier } = taskRecord
              //如果之前已经上传过则直接添加到附件列表
              let attachResponse = {
                  code: 0,
                  data: task
              }
              if (finished) {
                //之前已经上传过该附件直接秒传赋值就可以
                this.handleSuccess(attachResponse)
                return path
              } else {
                const errorList = await this.handleUpload(file, taskRecord, options)
                if (errorList.length > 0) {
                    this.$message.error("文件上传错误");
                    return;
                }
                let upLoadRes = await preMergeUpload(identifier)
                if (upLoadRes.code === 0) {
                    //上传完成
                    this.handleSuccess(attachResponse)
                    return path;
                } else {
                    this.$message.error("文件上传错误");
                }
              }
          } else {
              this.$message.error("文件上传错误");
          }
      },
      /**
       * 上传逻辑处理，如果文件已经上传完成（完成分块合并操作），则不会进入到此方法中
       */
      handleUpload(file, taskRecord, options) {
        let lastUploadedSize = 0; // 上次断点续传时上传的总大小
        let uploadedSize = 0 // 已上传的大小
        const totalSize = file.size || 0 // 文件总大小
        let startMs = new Date().getTime(); // 开始上传的时间
        const { exitPartList, chunkSize, chunkNum, fileIdentifier } = taskRecord

        // 获取从开始上传到现在的平均速度（byte/s）
        const getSpeed = () => {
            // 已上传的总大小 - 上次上传的总大小（断点续传）= 本次上传的总大小（byte）
            const intervalSize = uploadedSize - lastUploadedSize
            const nowMs = new Date().getTime()
            // 时间间隔（s）
            const intervalTime = (nowMs - startMs) / 1000
            return intervalSize / intervalTime
        }
        const uploadNext = async (partNumber) => {
            const start = new Number(chunkSize) * (partNumber - 1)
            const end = start + new Number(chunkSize)
            const blob = file.slice(start, end)
            const { code, data, msg } = await preSignUrl({ identifier: fileIdentifier, partNumber: partNumber })
            if (code === 0 && data) {
            await axios.request({
                url: data,
                method: 'PUT',
                data: blob,
                headers: { 'Content-Type': 'application/octet-stream' }
            })
            return Promise.resolve({ partNumber: partNumber, uploadedSize: blob.size })
            }
            return Promise.reject(`分片${partNumber}， 获取上传地址失败`)
        }
        /**
         * 更新上传进度
         * @param increment 为已上传的进度增加的字节量
         */
        const updateProcess = (increment) => {
            increment = new Number(increment)
            const { onProgress } = options
            let factor = 1000; // 每次增加1000 byte
            let from = 0;
            // 通过循环一点一点的增加进度
            while (from <= increment) {
            from += factor
            uploadedSize += factor
            //百分比与 100 进行比较，取较小的值   更新进度
            const percent = Math.min((100, Number(Math.round(uploadedSize / totalSize * 100))))
            this.percentageD = percent ? percent : 0
            console.log(this.percentageD,'上传进度');
            onProgress({ percent: percent })
            }
            const speed = getSpeed();
            const remainingTime = speed != 0 ? Math.ceil((totalSize - uploadedSize) / speed) + 's' : '未知'
            console.log('剩余大小：', (totalSize - uploadedSize) / 1024 / 1024, 'mb');
            console.log('当前速度：', (speed / 1024 / 1024).toFixed(2), 'mbps');
            console.log('预计完成：', remainingTime);
        }
        return new Promise(resolve => {
            const failArr = [];
            const queue = Queue(5, {
            "retry": 3, //Number of retries
            "retryIsJump": false, //retry now?
            "workReject": function (reason, queue) {
                failArr.push(reason)
            },
            "queueEnd": function (queue) {
                resolve(failArr);
            }
            })
            this.fileUploadChunkQueue[file.uid] = queue
            // this.fileParsingIs = false;
            for (let partNumber = 1; partNumber <= chunkNum; partNumber++) {
            const exitPart = (exitPartList || []).find(exitPart => exitPart.partNumber == partNumber)
            if (exitPart) {
                // 分片已上传完成，累计到上传完成的总额中,同时记录一下上次断点上传的大小，用于计算上传速度
                lastUploadedSize += new Number(exitPart.size)
                updateProcess(exitPart.size)
            } else {
                queue.push(() => uploadNext(partNumber).then(res => {
                // 单片文件上传完成再更新上传进度
                updateProcess(res.uploadedSize)
                }))
            }
            }
            if (queue.getLength() == 0) {
            // 所有分片都上传完，但未合并，直接return出去，进行合并操作
            resolve(failArr);
            return;
            }
            queue.start()
        })
      },
      /**
       * 获取一个上传任务，没有则初始化一个
       */
      async getTaskInfo(file) {
        let task;
        const identifier = await md5(file);
        const { code, data, msg } = await getTaskInfoUpload(identifier);
        if (code === 0) {
            task = data;
            if (!task) {
              const initTaskData = {
                  identifier,
                  fileName: file.name,
                  totalSize: file.size,
                  chunkSize: 10 * 1024 * 1024,
              };
              const { code, data, msg } = await initTaskUpload(initTaskData);
              if (code === 0) {
                  task = data;
              } else {
                  this.$message.error("文件上传错误");
              }
            }
        } else {
            this.$message.error("文件上传错误");
        }
        return task;
      },
      //上传成功
      handleSuccess(response, file, fileList) {
        if (response.code === 0) {
          setTimeout(() => {
            this.disabledUplaod = false;
            this.fileParsingIs = false;
            this.closeFlag = false;
            this.isSuccessFlag = true;
          }, 800);
          let item = response.data.path;
          this.modelForm.fileUpload = item;
        } else {
          fileList.pop();
        }
      },
      handleAvatarSuccess(res, file){
        this.modelForm.fileUpload = res;
        // 上传成功后，手动验证一次表单
        this.$refs.modelForm.validateField('fileUpload');
      },
        handleError(file) {
          let uid = file.uid // 去除文件列表失败文件
          let idx = this.$refs.uploadRef.uploadFiles.findIndex(item => item.uid === uid) 
          this.$refs.uploadRef.uploadFiles.splice(idx, 1) 
      },
      handleProgress(event, file, fileList) {
        // 强制更新 DOM
        this.$forceUpdate();
      },
      handleExceed(files, fileList) {
        this.$message.warning(`只能上传一个文件`);
      },
      // handleRemove(files, fileList) {
      //   this.$refs.uploadRef.clearFiles();
      //   this.fileParsingIs = false;
      //   this.closeFlag = false;
      //   this.isSuccessFlag = false;
      // },
      clearFileFun(){
        this.$refs.uploadRef.clearFiles();
        this.fileParsingIs = false;
        this.closeFlag = false;
        this.isSuccessFlag = false;
      }
    }
};
</script>
<style lang="scss" scoped>
/* 隐藏默认的清空和成功按钮 */
::v-deep{
  .upload-demo{
    .el-upload-list__item-status-label,.el-icon-close,.el-icon-close-tip{
      display: none !important;
    }
    .el-upload-list__item .el-progress__text {
      position: absolute;
      right: 24px !important;
      top: -13px;
    }
  }
}
.wrapForm {
  .demo-ruleForm {
    padding-bottom: 80px;
    .el-form-item {
      margin-bottom: 30px !important;
    }
  }
  .btnWarp {
    text-align: center;
  }
  .fileTips{
    font-size: 12px;
    color: #545559;
    position: absolute;
    right: 40px;
    top: 42px;
  }
  .actionIcon{
    position: absolute;
    right: 2px;
    top: 50px;
    display: flex;
  }
  .upload-demo {
    .uploadContent {
      display: flex;
      justify-content: space-between;
      .uploadInp {
        display: flex;
        border: 1px solid #ededed;
        padding: 0px 5px;
        justify-content: center;
        align-items: center;
        margin-right: 35px;
        .el-upload__text {
          margin-left: 15px;
        }
      }
      .el-upload__tip {
        margin-top: 15px;
      }
    }
  }
}
</style>

md5代码,新建一个md5的文件

import SparkMD5 from 'spark-md5'
import { Loading } from 'element-ui';
const DEFAULT_SIZE = 20 * 1024 * 1024
const md5 = (file, chunkSize = DEFAULT_SIZE) => {
  return new Promise((resolve, reject) => {
    const startMs = new Date().getTime();
    // const loading = Loading.service({
    //   lock: true,
    //   text: '处理中，请稍后！',
    //   spinner: 'el-icon-loading',
    //   background: 'rgba(0, 0, 0, 0.7)'
    // });
    let blobSlice =
      File.prototype.slice ||
      File.prototype.mozSlice ||
      File.prototype.webkitSlice;
    let chunks = Math.ceil(file.size / chunkSize);
    // console.log("file.size::: ", file.size);
    let currentChunk = 0;
    let spark = new SparkMD5.ArrayBuffer(); //追加数组缓冲区。
    let fileReader = new FileReader(); //读取文件
    fileReader.onload = function (e) {
      spark.append(e.target.result);
      currentChunk++;
      if (currentChunk < chunks) {
        loadNext();
      } else {
        const md5 = spark.end(); //完成md5的计算，返回十六进制结果。
        console.log('文件md5计算结束，总耗时：', (new Date().getTime() - startMs) / 1000, 's')
        // loading.close()
        resolve(md5);

      }
    };
    fileReader.onerror = function (e) {
      // loading.close()
      reject(e);
    };

    function loadNext() {
      console.log('当前part number：', currentChunk, '总块数：', chunks);
      let start = currentChunk * chunkSize;
      let end = start + chunkSize;
      (end > file.size) && (end = file.size);
      fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
    }
    loadNext();
  });
}

export default md5

用到的接口

/**分片上传接口开始 */ 
/**
 * 根据文件的md5获取未上传完的任务
 * @param identifier 文件md5
 * @returns {Promise<AxiosResponse<any>>}
 */
export const getTaskInfoUpload = (identifier) => {
    return request({
        url: `${wrjURL.URL}/common/file/taskInfo?identifier=${identifier}`,
        method: "get",
        // params,
    });
};
/**
 * 初始化一个分片上传任务
 * @param identifier 文件md5
 * @param fileName 文件名称
 * @param totalSize 文件大小
 * @param chunkSize 分块大小
 * @returns {Promise<AxiosResponse<any>>}
 */
export const initTaskUpload = ({identifier, fileName, totalSize, chunkSize}) => {
    return request({
        url: `${wrjURL.URL}/common/file/initTask`,
        method: "post",
        data:{identifier, fileName, totalSize, chunkSize}
    });
};
/**
 * 获取预签名分片上传地址
 * @param identifier 文件md5
 * @param partNumber 分片编号
 * @returns {Promise<AxiosResponse<any>>}
 */
 export const preSignUrl = ({ identifier, partNumber }) => {
    return request({
        url: `${wrjURL.URL}/common/file/preSignUploadUrl?identifier=${identifier}&partNumber=${partNumber}`,
        method: "get",
        // params,
    });
};
/**
 * 根据文件的md5获取未上传完的任务
 * @param identifier 文件md5
 * @returns {Promise<AxiosResponse<any>>}
 */
 export const preMergeUpload = (identifier) => {
    return request({
        url: `${wrjURL.URL}/common/file/merge?identifier=${identifier}`,
        method: "get",
        // params,
    });
};
/**分片上传接口结束 */