后台合并文件时，解决了并发合并冲突异常的情况。测试了2G文件上传，测试了4G文件上传，测试了8G文件上传，都成功了
前端代码，单文件上传
@{
ViewBag.Title = "Upload";
}
<script src="~/Scripts/jquery-3.4.1.min.js"></script>
<h2>文件上传</h2>
<div>
<input type="file" id="file" style="width:78%;" />
<input type="button" id="btnDoUpload" value="执行上传" />
</div>
<div id="fileInfo"></div>
<div>
<span>上传进度：</span>
<b id="progressStrip"></b>
</div>
<script type="text/javascript">
var fileMyUploder = {
//每批次最大发送请求数，设置太大浏览器会报异常net::ERR_INSUFFICIENT_RESOURCES
concurrentNum: 200,
//上传成功的计数
succeed: 0,
//发送的文件分片下标
forIndex: 0,
//总分片数
shardCount: 0,
//每批次成功数
eachIndex: 0,
//请求id
requestId: (new Date().getTime() / 1000).toFixed(0),
uploadUrl: '',//上传后台处理地址
doneFunc: function () { },//上传成功后回调方法
sendData: function (i, file, shardSize, size, Count, name) {
//计算每一片的起始与结束位置
var start = i * shardSize,
end = Math.min(size, start + shardSize);
//构造一个表单，FormData是HTML5新增的
var index = i + 1;
var form = new FormData();
form.append("data", file.slice(start, end)); //slice方法用于切出文件的一部分
form.append("name", this.requestId +"_"+ name);
form.append("total", Count); //总片数
form.append("index", index); //当前是第几片
$("#progressStrip").html(0 + " / " + fileMyUploder.shardCount);
//Ajax提交
$.ajax({
//url: "/Home/Upload?index=" + index,
url: fileMyUploder.uploadUrl + "?index=" + index,
type: "POST",
data: form,
async: true, //异步
processData: false, //很重要，告诉jquery不要对form进行处理
contentType: false, //很重要，指定为false才能形成正确的Content-Type
success: function (s) {
//alert(s);
if (s.Code == 1) {
//alert("上传失败！");
console.log(s.Msg);
} else {
++fileMyUploder.succeed;
++fileMyUploder.eachIndex;
if (fileMyUploder.eachIndex >= fileMyUploder.concurrentNum && fileMyUploder.succeed <= fileMyUploder.shardCount) {
console.log("fileMyUploder.succeed=" + fileMyUploder.succeed);
console.log("fileMyUploder.eachIndex=" + fileMyUploder.eachIndex);
//剩余数
var remainder = Count - fileMyUploder.succeed;
console.log("remainder=" + remainder);
//console.log("Count=" + Count + ",shardCount=" + fileMyUploder.shardCount);
var xhSize = remainder > fileMyUploder.concurrentNum ? fileMyUploder.concurrentNum : remainder;
console.log("xhSize=" + xhSize);
console.log("forIndex=" + fileMyUploder.forIndex);
for (var i = 0; i < xhSize; ++fileMyUploder.forIndex) {
fileMyUploder.sendData(fileMyUploder.forIndex, file, shardSize, size, Count, name);
i++;
}
fileMyUploder.eachIndex = 0;
}
}
$("#progressStrip").html(fileMyUploder.succeed + " / " + fileMyUploder.shardCount);
if (fileMyUploder.succeed == fileMyUploder.shardCount) {
//console.log(s);
fileMyUploder.doneFunc(s);//执行上传成功回调
fileMyUploder.succeed = 0;
fileMyUploder.forIndex = 0;
fileMyUploder.shardCount = 0;
fileMyUploder.eachIndex = 0;
$("#progressStrip").html("完成");
}
},
error: function (e) {
console.log(e);
$("#btnDoUpload").removeAttr("disabled");
//alert(e);
if (e.Code == 1) {
//alert("上传失败！");
}
}
});
},
upload: function () {
fileMyUploder.succeed = 0;
fileMyUploder.forIndex = 0;
fileMyUploder.shardCount = 0;
fileMyUploder.eachIndex = 0;
var shardSize = 2 * 1024 * 1024; //以2MB为一个分片
for (var i = 0; i < $("#file")[0].files.length; i++) {
fileMyUploder.shardCount += Math.ceil($("#file")[0].files[i].size / shardSize); //总片数
}
for (var j = 0; j < $("#file")[0].files.length; j++) {
var file = $("#file")[0].files[j], //文件对象
name = file.name, //文件名
size = file.size; //总大小
var Count = Math.ceil(file.size / shardSize);
var xhSize = Count > fileMyUploder.concurrentNum ? fileMyUploder.concurrentNum : Count;
for (var i = 0; i < xhSize; ++fileMyUploder.forIndex) {
fileMyUploder.sendData(fileMyUploder.forIndex, file, shardSize, size, Count, name);
i++;
}
}
}
};
fileMyUploder.uploadUrl = "/file/DoUpload";//上传后台处理地址
fileMyUploder.doneFunc = function (d) {
//上传成功后回调函数
console.log("上传成功后回调函数:");
console.log(d);
$("#btnDoUpload").removeAttr("disabled");
//var file = $("#file")[0].files[0];//文件对象
//console.log(file);
//var fileId = p.uuid();
//var fileSizeKb = (file.size / 1014).toFixed(1);
//var htmlArr = ['<tr id="' + fileId + '" fileSizeKb="' + fileSizeKb + '" OriginalFileName="' + file.name + '" durl="' + d.dbFileUrl + '">'
// , '<td>' + file.name + '</td>'
// , '<td>' + fileSizeKb + 'kb</td>'
// , '<td style="color: #5FB878;">上传成功</td>'
// , '<td>'
// , '<button class="layui-btn layui-btn-xs layui-btn-danger demo-delete">删除</button>'
// , '</td>'
// , '</tr>'];
如果选的图片则显示出来
//var imageType = /image.*/;
//if (file.type.match(imageType)) {
// var reader = new FileReader();
// reader.onload = function () {
// var img = new Image();
// img.src = reader.result;
// //console.log(img);
// //console.log(img.outerHTML);
// $("#" + fileId + " td:first").html(img.outerHTML);
// };
// reader.readAsDataURL(file);
//}
//$("#demoList").append(htmlArr.join(''));
删除绑定
//$(".demo-delete").click(function () {
// $(this).parent().parent().remove();
//});
}
$("#btnDoUpload").click(function () {
$("#btnDoUpload").attr("disabled", "disabled");
fileMyUploder.upload();
});
$("#file").on("change", function () {
var fileInfo = $("#file")[0].files[0];
var kb = parseFloat(fileInfo.size / 1024).toFixed(2) ;
console.log("文件大小kb=" + kb);
if (kb < 1024) {
$("#fileInfo").html("文件大小：" + kb + " KB");
return;
}
var mb = parseFloat(kb / 1024).toFixed(2) ;
console.log("文件大小MB=" + mb);
if (mb < 1024) {
$("#fileInfo").html("文件大小：" + mb + " MB");
return;
}
var gb = parseFloat(mb / 1024).toFixed(2) ;
$("#fileInfo").html("文件大小：" + gb + " GB");
});
</script>
单文件上传后台接收
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using System.Web.Mvc;
using WebApplicationFileSys.Filter;
using WebApplicationFileSys.Models;
namespace WebApplicationFileSys.Controllers
{
/// <summary>
/// 文件上传处理
/// </summary>
/// 添加时间：2023-8-31 18:00:46
public class FileController : Controller
{
// 视图，上传页面
[LoginFilter(Roles = "admin")]
public ActionResult Upload()
{
return View();
}
/// <summary>
/// 并发合并检查
/// </summary>
static ConcurrentDictionary<string, string> checkMergeFile = new ConcurrentDictionary<string, string>();
await Task.Yied
//文件上传，处理
[HttpPost]
[LoginFilter(Roles = "admin")]
public async Task<ActionResult> DoUpload()
{
//从Request中取参数，注意上传的文件在Requst.Files中
string name = Request["name"];
int total = Convert.ToInt32(Request["total"]);
int index = Convert.ToInt32(Request["index"]);
var data = Request.Files["data"];
await Task.Yield();
try
{
//保存一个分片到磁盘上
string dirRoot = Server.MapPath("~/Upload");
string extend = Path.GetExtension(name).ToString();
//普通文件保存目录
string dir = Path.Combine(dirRoot, DateTime.Now.ToString("yyyy_MM"));
//图片保存目录
if (".jpg^.jpeg^.png^.gif^.bmp^.ico^.webp".Contains(extend))
{
dir = Path.Combine(dirRoot, "img", DateTime.Now.ToString("yyyy_MM"));
}
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string file = Path.Combine(dir, name + "_" + index);
data.SaveAs(file);
//最后一个分片文件，开始合并
if (total == index)
{
Task.Run(() =>
{
//判断是否已经有此文件在合并了，处理并发合并情况
bool isMerge = checkMergeFile.Values.Contains(name);
if (isMerge)
{
LogHelpter.AddLog("阻止了一次并发合并文件", null, "Merge_conflict");
return;
}
//等待文件数 对齐总分片
while (true)
{
string[] files2 = Directory.GetFiles(dir, name + "_*");
LogHelpter.AddLog($"while----{name}，total=" + total + "，接收文件数=" + files2.Length);
if (files2.Length == total)
{
break;
}
System.Threading.Thread.Sleep(1000);
}
//计时
System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
string chKey = Guid.NewGuid().ToString("N");
checkMergeFile.TryAdd(chKey, name);
LogHelpter.AddLog("checkMergeFile个数" + checkMergeFile.Count + ",开始合并...");
file = Path.Combine(dir, name);
var fs = new FileStream(file, FileMode.Create);
for (int i = 1; i <= total; ++i)
{
string part = Path.Combine(dir, name + "_" + i);
var bytes = System.IO.File.ReadAllBytes(part);
fs.Write(bytes, 0, bytes.Length);
bytes = null;
System.IO.File.Delete(part);
}
fs.Close();
fs.Dispose();
string outValue = string.Empty;
checkMergeFile.TryRemove(chKey, out outValue);
stopwatch.Stop();
LogHelpter.AddLog($"{name} 合并耗时（毫秒）{stopwatch.ElapsedMilliseconds}");
});
//数据库保存相对路径，生产环境一般存英文， 比如/upload/123444.jpg
//string dbFileUrl = "/" + file.Replace(AppDomain.CurrentDomain.BaseDirectory, "").Replace(@"\", "/");
string dbFileUrl = "/" + file.Replace(AppDomain.CurrentDomain.BaseDirectory, "").Replace(@"\", "/").Replace("_" + index, "");
//最后一个分片，返回相对url，用于数据库保存，
return Json(new { Code = 0, Msg = "上传成功", url = dbFileUrl });
}
//返回是否成功，此处做了简化处理
return Json(new { Code = 0, Msg = "上传成功" });
}
catch (Exception ex)
{
return Json(new { Code = 1, Msg = ex.ToString() });
}
}
}
}
前端代码方式二
<input type="file" id="file" accept=".pdf,.jpg,.jpeg,.png,.gif,.bmp,.webp,.doc,.docx,.xls,.xlsx,.rtf,.ppt,.pptx,.zip,.rar" />
<a href="javascript:;" class="button org-btn small" id="addFile">上传</a>
<span>支持的文件格式：@string.Join(" ", CommonData.AllowFileUploadArr)</span>
<div>上传进度：<span id="progressStrip"></span></div>
<table class="layui-table">
<thead>
<tr>
<th>文件名</th>
<th>大小</th>
<th>状态</th>
<th>操作</th>
</tr>
</thead>
<tbody id="demoList"></tbody>
</table>
var fileMyUploder = {
//每批次最大发送请求数，设置太大浏览器会报异常net::ERR_INSUFFICIENT_RESOURCES
concurrentNum: 200,
//上传成功的计数
succeed: 0,
//发送的文件分片下标
forIndex: 0,
//总分片数
shardCount: 0,
//每批次成功数
eachIndex:0,
//请求id
requestId:(new Date().getTime()/1000).toFixed(0),
uploadUrl:'',//上传后台处理地址
doneFunc:function(){},//上传成功后回调方法
sendData: function (i, file, shardSize, size, Count, name) {
//计算每一片的起始与结束位置
var start = i * shardSize,
end = Math.min(size, start + shardSize);
//构造一个表单，FormData是HTML5新增的
var index = i + 1;
var form = new FormData();
form.append("data", file.slice(start, end)); //slice方法用于切出文件的一部分
form.append("name", this.requestId+name);
form.append("total", Count); //总片数
form.append("index", index); //当前是第几片
$("#progressStrip").html(0 + " / " + fileMyUploder.shardCount);
//Ajax提交
$.ajax({
//url: "/Home/Upload?index=" + index,
url:fileMyUploder.uploadUrl+"?index=" + index,
type: "POST",
data: form,
async: true, //异步
processData: false, //很重要，告诉jquery不要对form进行处理
contentType: false, //很重要，指定为false才能形成正确的Content-Type
success: function (s) {
//alert(s);
if (s.Code == 1) {
//alert("上传失败！");
console.log(s.Msg);
} else {
++fileMyUploder.succeed;
++fileMyUploder.eachIndex;
if (fileMyUploder.eachIndex >= fileMyUploder.concurrentNum && fileMyUploder.succeed <= fileMyUploder.shardCount) {
console.log("fileMyUploder.succeed=" + fileMyUploder.succeed);
console.log("fileMyUploder.eachIndex=" + fileMyUploder.eachIndex);
//剩余数
var remainder = Count - fileMyUploder.succeed;
console.log("remainder="+remainder);
//console.log("Count=" + Count + ",shardCount=" + fileMyUploder.shardCount);
var xhSize = remainder > fileMyUploder.concurrentNum ? fileMyUploder.concurrentNum : remainder;
console.log("xhSize=" + xhSize);
console.log("forIndex=" + fileMyUploder.forIndex);
for (var i=0; i < xhSize; ++fileMyUploder.forIndex) {
fileMyUploder.sendData(fileMyUploder.forIndex, file, shardSize, size, Count, name);
i++;
}
fileMyUploder.eachIndex = 0;
}
}
$("#progressStrip").html(fileMyUploder.succeed + " / " + fileMyUploder.shardCount);
if (fileMyUploder.succeed == fileMyUploder.shardCount) {
//console.log(s);
fileMyUploder.doneFunc(s);//执行上传成功回调
fileMyUploder.succeed = 0;
fileMyUploder.forIndex = 0;
fileMyUploder.shardCount = 0;
fileMyUploder.eachIndex = 0;
$("#progressStrip").html("完成");
}
},
error: function (e) {
console.log(e);
//alert(e);
if (e.Code == 1) {
//alert("上传失败！");
}
}
});
},
upload: function () {
fileMyUploder.succeed = 0;
fileMyUploder.forIndex = 0;
fileMyUploder.shardCount = 0;
fileMyUploder.eachIndex = 0;
var shardSize = 2 * 1024 * 1024; //以2MB为一个分片
for (var i = 0; i < $("#file")[0].files.length; i++) {
fileMyUploder.shardCount += Math.ceil($("#file")[0].files[i].size / shardSize); //总片数
}
for (var j = 0; j < $("#file")[0].files.length; j++) {
var file = $("#file")[0].files[j], //文件对象
name = file.name, //文件名
size = file.size; //总大小
var Count = Math.ceil(file.size / shardSize);
var xhSize = Count > fileMyUploder.concurrentNum ? fileMyUploder.concurrentNum : Count;
for (var i=0; i < xhSize; ++fileMyUploder.forIndex) {
fileMyUploder.sendData(fileMyUploder.forIndex, file, shardSize, size, Count, name);
i++;
}
}
}
};
fileMyUploder.uploadUrl = "/file/UploadBigFile";//上传后台处理地址
fileMyUploder.doneFunc = function (d) {
//上传成功后回调函数
console.log("上传成功后回调函数:");
console.log(d);
var file = $("#file")[0].files[0];//文件对象
console.log(file);
var fileId = p.uuid();
var fileSizeKb = (file.size / 1014).toFixed(1);
var htmlArr = ['<tr id="' + fileId + '" fileSizeKb="' + fileSizeKb + '" OriginalFileName="' + file.name + '" durl="' + d.dbFileUrl+'">'
, '<td>' + file.name + '</td>'
, '<td>' + fileSizeKb + 'kb</td>'
, '<td style="color: #5FB878;">上传成功</td>'
, '<td>'
, '<button class="layui-btn layui-btn-xs layui-btn-danger demo-delete">删除</button>'
, '</td>'
, '</tr>'];
// 如果选的图片则显示出来
var imageType = /image.*/;
if (file.type.match(imageType)) {
var reader = new FileReader();
reader.onload = function () {
var img = new Image();
img.src = reader.result;
//console.log(img);
//console.log(img.outerHTML);
$("#" + fileId + " td:first").html(img.outerHTML);
};
reader.readAsDataURL(file);
}
$("#demoList").append(htmlArr.join(''));
//删除绑定
$(".demo-delete").click(function () {
$(this).parent().parent().remove();
});
}
$("#addFile").click(function () {
fileMyUploder.upload();
});
代码方式二，后台代码：
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Web;
using System.Web.Mvc;
using WebApplicationFileUpload.Models;
namespace WebApplicationFileUpload.Controllers
{
public class HomeController : Controller
{
public ActionResult Index()
{
return View();
}
/// <summary>
/// 并发合并检查
/// </summary>
static ConcurrentDictionary<string, string> checkMergeFile = new ConcurrentDictionary<string, string>();
[HttpPost]
public ActionResult Upload()
{
//从Request中取参数，注意上传的文件在Requst.Files中
string name = Request["name"];
int total = Convert.ToInt32(Request["total"]);
int index = Convert.ToInt32(Request["index"]);
var data = Request.Files["data"];
try
{
//保存一个分片到磁盘上
string dir = Server.MapPath("~/Upload");
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string file = Path.Combine(dir, name + "_" + index);
data.SaveAs(file);
if (total - index < 20)
{
string[] files = Directory.GetFiles(dir,name + "_*");
LogHelpter.AddLog("total=" + total+ "，files.Length="+ files.Length);
//如果已经是最后一个分片，组合
//当然你也可以用其它方法比如接收每个分片时直接写到最终文件的相应位置上，但要控制好并发防止文件锁冲突
if (files.Length == total)
{
//判断是否已经有此文件在合并了，处理并发合并情况
bool isMerge = checkMergeFile.Values.Contains(name);
if (isMerge)
{
LogHelpter.AddLog("阻止了一次并发合并文件");
return Json(new { Code = 0, Msg = "上传成功" });
}
string chKey = Guid.NewGuid().ToString("N");
checkMergeFile.TryAdd(chKey, name);
LogHelpter.AddLog("checkMergeFile个数" + checkMergeFile.Count);
System.Threading.Thread.Sleep(100);
file = Path.Combine(dir, name);
var fs = new FileStream(file, FileMode.Create);
for (int i = 1; i <= total; ++i)
{
string part = Path.Combine(dir, name + "_" + i);
var bytes = System.IO.File.ReadAllBytes(part);
fs.Write(bytes, 0, bytes.Length);
bytes = null;
System.IO.File.Delete(part);
}
fs.Close();
fs.Dispose();
string outValue = string.Empty;
checkMergeFile.TryRemove(chKey, out outValue);
}
}
//数据库保存相对路径，生产环境一般存英文， 比如/upload/123444.jpg
string dbFileUrl = "/"+file.Replace(AppDomain.CurrentDomain.BaseDirectory,"").Replace(@"\", "/");
//返回是否成功，此处做了简化处理
return Json(new { Code = 0, Msg = "上传成功",url= dbFileUrl });
}
catch (Exception ex)
{
return Json(new { Code = 1, Msg = ex.ToString() });
}
}
}
}
参考了此文：
https://www.cnblogs.com/caijiabao/p/9662988.html
2181
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
