大文件分片上传并发

我这边使用的是boostrap-fileimput

初始化文件上传框

$(document).ready(function () {
        $("#file-upload_import").fileinput({
            uploadUrl: "#",
            language: "zh", //设置语言
            showPreview: true,
            autoReplace: true,
            // uploadUrl: "/uact/uploadfile", //上传的地址
            uploadExtraData: { type: "1012-0001" },
            allowedFileExtensions: ["json"], //接收的文件后缀,
            allowedPreviewTypes: ["image"], //可以预览的文件类型
            // initialPreviewAsData: File,
            // maxFileSize:10485760, //10G
            showUpload: false,
            maxFileCount: 1,
            minFileCount: 1,
            previewTemplates: {
                generic: '<div class="file-preview-frame" id="{previewId}" data-fileindex="{fileindex}" data-template="{template}">\n' +
                    '</div>\n'
            },
            dropZoneTitle: "请选择一个json文件上传",
            // validateInitialCount: true,
            // showCaption: true,
            fileActionSettings: {
                // 在预览窗口中为新选择的文件缩略图设置文件操作的对象配置
                showRemove: true, // 显示删除按钮
                showUpload: false, // 显示上传按钮
                showDownload: false, // 显示下载按钮
                showZoom: false, // 显示预览按钮
                showDrag: false, // 显示拖拽
            },
        });
    })

分片和重传

var succeed = 0;
var fail_parts = []

async function progress(percent, $element) {
    var progressBarWidth = percent * $element.width() / 100;
    $element.find('div').animate({ width: progressBarWidth }, 500).html(percent + "% ");
}

function updateProgress(percentage) {
    $('.progress .progress-bar').attr('data-transitiongoal', percentage).progressbar({ display_text: 'fill' });
}

function generateUUID() {
    // 生成全局uuid
    var d = new Date().getTime();
    if (window.performance && typeof window.performance.now === "function") {
        d += performance.now();
    }
    var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
        var r = (d + Math.random() * 16) % 16 | 0;
        d = Math.floor(d / 16);
        return (c == "x" ? r : (r & 0x3) | 0x8).toString(16);
    },)

    return uuid;
}

const form = document.getElementById('upload-form');
const fileInput = document.getElementById('file-upload_import');
form.addEventListener('submit', async (e) => {
    e.preventDefault();
    $("#upload").attr("disabled", "disabled");
    const file = fileInput.files[0];
    if (!file) {
        alert("至少选择一个文件")
        return
    }

    const chunkSize = 10 * 1024 * 1024; // 10MB 分片大小
    // const chunkSize = 1 * 1024; // 1KB 分片大小
    let upload_id = generateUUID()
    // 切割文件
    const chunks = [];
    let offset = 0;
    while (offset < file.size) {
        const chunk = file.slice(offset, offset + chunkSize);
        chunks.push(chunk);
        offset += chunkSize;
    }

    $('.progress .progress-bar').attr('data-transitiongoal', 0).progressbar({ display_text: 'fill' });
    // 逐个上传分片
    const urls = [];
    for (let i = 0; i < chunks.length; i++) {
        const formData = new FormData();
        formData.append('chunk', chunks[i]);
        formData.append("upload_id", upload_id)
        formData.append('index', i);
        formData.append('size', file.size)
        urls.push({
            url: "你的分片上传地址",
                param: formData
        })
    }
    try {
        concurRequest(urls, 6).then(ret => {
            setTimeout(() => {
                $(document).trigger('chunksUploaded', [fail_parts, chunks, upload_id]);
            }, 1000);
        })
    } catch (error) {
        console.log("分片上传错误:" + error.data[0].index)
        fail_parts.push(error.data[0]);
    }
});


$(document).on('chunksUploaded', async function (event, fail_parts, chunks, upload_id) {
    // 重传失败分片
    if (fail_parts.length > 0) {
        let retry_urls = [];
        for (let ele of fail_parts) {
            const formData = new FormData();
            formData.append('chunk', chunks[ele.index]);
            formData.append("upload_id", ele.upload_id)
            formData.append('retry', true)
            formData.append('index', ele.index)
            retry_urls.push({
                url: "你的分片上传地址",
                param: formData
            })
        }
        try {
            concurRequest(retry_urls, 3).then(ret => {
                console.log("重传分片成功！",res);
            })
        } catch (error) {
            console.log("重传分片" + error.msg + ":" + part.index)
        }
    }

    setTimeout(() => {
        $("#upload").removeAttr("disabled");
        if (fail_parts.length == 0) {
            const formData = new FormData();
            formData.append('total_chunks', chunks.length);
            formData.append("upload_id", upload_id);
            $.ajax({
                url: '/mongodb/import/merge/',
                method: 'POST',
                data: formData,
                processData: false,
                contentType: false,
                success: function (data) {
                    if (data.status === 0) {
                        alert('文件上传成功！');
                    }
                }
            })
        } else {
            alert('文件上传失败,请稍后重新上传');
        }
    }, 2000);

})

并发方法

/**
* 并发请求
* @param {string[]} urls 待请求的url数组
* @param {number} maxNum 最大的并发数
*/
function concurRequest(urls, maxNum) {
    return new Promise(resolve => {
        if (urls.length === 0) {
            /* 先考虑边界问题 */
            resolve([]);
            return;
        }
        const results = [] //存储请求结果并返回
        let index = 0; //下一个请求的url下标
        let count = 0; //计算请求的数量

        async function request() {
            if (index === urls.length) {
                return;
            }
            const i = index;
            const url = urls[index].url;
            const res = urls[index].param;
            index++
            try {
                let ret = await fetch(url, {
                    method: "POST",
                    body: res,
                });
                results[i] = ret //不能用push，因为这样就会出现异步完成优先写入不符合要求。
                var percent = ((index / urls.length).toFixed(2)) * 100;
                updateProgress(percent);
            } catch (error) {
                results[i] = err
            } finally {
                count++
                if (count === urls.length) {
                    resolve(results);
                }
                 request();
            }
        }

        const times = Math.min(maxNum, urls.length);
        for (let i = 0; i < times; i++) {
            request();
        }

    })
}