-
属于专栏-前端项目难点
-
重要程度:★★★★★
当我们在做文件的导入功能的时候,如果导入的文件过大,可能会导所需要的时间够长,且失败后需要重新上传,我们需要前后端结合的方式解决这个问题
我们需要做几件事情如下:
-
对文件做切片,即将一个请求拆分成多个请求,每个请求的时间就会缩短,且如果某个请求失败,只需要重新发送这一次请求即可,无需从头开始
-
通知服务器合并切片,在上传完切片后,前端通知服务器做合并切片操作
-
控制多个请求的并发量,防止多个请求同时发送,造成浏览器内存溢出,导致页面卡死
-
做断点续传,当多个请求中有请求发送失败,例如出现网络故障、页面关闭等,我们得对失败的请求做处理,让它们重复发送
前端
示例代码仓库
步骤1-切片,合并切片
在JavaScript中
,文件FIle
对象是Blob
对象的子类,Blob
对象包含一个重要的方法slice
通过这个方法,我们就可以对二进制文件进行拆分,具体代码如下:
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=s, initial-scale=1.0">
<title>Document</title>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
</head>
<body>
<input type="file" id="fileInput">
<button id="uploadBtn">上传</button>
</body>
<script>
axios.defaults.baseURL = 'http://localhost:3000'
var file = null
document.getElementById('fileInput').onchange = function({target: {files}}){
file = files[0]
}
document.getElementById('uploadBtn').onclick = async function(){
if (!file) return
let size = 1024 * 50
let fileChunks = []
let index = 0
for(let cur = 0; cur < file.size; cur += size){
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size)
})
}
const uploadList = fileChunks.map((item, index) => {
let formData = new FormData()
formData.append('filename', file.name)
formData.append('hash', item.hash)
formData.append('chunk', item.chunk)
return axios({
method: 'post',
url: '/upload',
data: formData
})
})
await Promise.all(uploadList)
await axios({
method: 'get',
url: '/merge',
params: {
filename: file.name
}
});
console.log('上传完成')
}
</script>
</html>
步骤2-并发控制
结合Promise.race
和异步函数
实现,多个请求同时并发的数量,防止浏览器内存溢出,具体代码如下:
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=s, initial-scale=1.0">
<title>Document</title>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
</head>
<body>
<input type="file" id="fileInput">
<button id="uploadBtn">上传</button>
</body>
<script>
axios.defaults.baseURL = 'http://localhost:3000'
var file = null
document.getElementById('fileInput').onchange = function({target: {files}}){
file = files[0]
}
document.getElementById('uploadBtn').onclick = async function(){
if (!file) return
let size = 1024 * 50
let fileChunks = []
let index = 0
for(let cur = 0; cur < file.size; cur += size){
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size)
});
}
let pool = []
let max = 3
for(let i=0;i<fileChunks.length;i++){
let item = fileChunks[i]
let formData = new FormData()
formData.append('filename', file.name)
formData.append('hash', item.hash)
formData.append('chunk', item.chunk)
let task = axios({
method: 'post',
url: '/upload',
data: formData
})
task.then((data)=>{
let index = pool.findIndex(t=> t===task)
pool.splice(index)
})
pool.push(task)
if(pool.length === max){
await Promise.race(pool)
}
}
await axios({
method: 'get',
url: '/merge',
params: {
filename: file.name
}
});
console.log('上传完成')
}
</script>
</html>
步骤3-断点续传
在单个请求失败后,触发catch
的方法的时候,讲当前请求放到失败列表中,在本轮请求完成后,重复对失败请求做处理,具体代码如下:
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=s, initial-scale=1.0">
<title>Document</title>
<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>
</head>
<body>
<input type="file" id="fileInput">
<button id="uploadBtn">上传</button>
</body>
<script>
axios.defaults.baseURL = 'http://localhost:3000'
var file = null
document.getElementById('fileInput').onchange = function({target: {files}}){
file = files[0]
}
document.getElementById('uploadBtn').onclick = function(){
if (!file) return;
let size = 1024 * 50;
let fileChunks = [];
let index = 0
for(let cur = 0; cur < file.size; cur += size){
fileChunks.push({
hash: index++,
chunk: file.slice(cur, cur + size)
})
}
const uploadFileChunks = async function(list){
if(list.length === 0){
await axios({
method: 'get',
url: '/merge',
params: {
filename: file.name
}
});
console.log('上传完成')
return
}
let pool = []
let max = 3
let finish = 0
let failList = []
for(let i=0;i<list.length;i++){
let item = list[i]
let formData = new FormData()
formData.append('filename', file.name)
formData.append('hash', item.hash)
formData.append('chunk', item.chunk)
let task = axios({
method: 'post',
url: '/upload',
data: formData
})
task.then((data)=>{
let index = pool.findIndex(t=> t===task)
pool.splice(index)
}).catch(()=>{
failList.push(item)
}).finally(()=>{
finish++
if(finish===list.length){
uploadFileChunks(failList)
}
})
pool.push(task)
if(pool.length === max){
await Promise.race(pool)
}
}
}
uploadFileChunks(fileChunks)
}
</script>
</html>
后端
步骤1.安装依赖
npm i express@4.17.2
npm i multiparty@4.2.2
步骤2.接口实现
const express = require('express')
const multiparty = require('multiparty')
const fs = require('fs')
const path = require('path')
const { Buffer } = require('buffer')
const STATIC_FILES = path.join(__dirname, './static/files')
const STATIC_TEMPORARY = path.join(__dirname, './static/temporary')
const server = express()
server.use(express.static(path.join(__dirname, './dist')))
server.post('/upload', (req, res) => {
const form = new multiparty.Form();
form.parse(req, function(err, fields, files) {
let filename = fields.filename[0]
let hash = fields.hash[0]
let chunk = files.chunk[0]
let dir = `${STATIC_TEMPORARY}/${filename}`
try {
if (!fs.existsSync(dir)) fs.mkdirSync(dir)
const buffer = fs.readFileSync(chunk.path)
const ws = fs.createWriteStream(`${dir}/${hash}`)
ws.write(buffer)
ws.close()
res.send(`${filename}-${hash} 切片上传成功`)
} catch (error) {
console.error(error)
res.status(500).send(`${filename}-${hash} 切片上传失败`)
}
})
})
server.get('/merge', async (req, res) => {
const { filename } = req.query
try {
let len = 0
const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map((hash,index) => {
const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${index}`)
len += buffer.length
return buffer
});
const buffer = Buffer.concat(bufferList, len);
const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`)
ws.write(buffer);
ws.close();
res.send(`切片合并完成`);
} catch (error) {
console.error(error);
}
})
server.listen(3000, _ => {
console.log('http://localhost:3000/')
})
如果使用腾讯云
或阿里云
文件上传的服务,它们提供了npm
库,例如腾讯云
的cos-js-sdk-v5[3],它自身提供的切片相关的配置