原因
想实现断点续传，特别是网络不畅的情况下适用。
使用方法
效果图 ：
使用方法：
<div id="chunkupload"></div>
<script>
var options = {
accept:['enc'],
maxSize:10*1024*1024*1024,
chunkSize:5*1024*1024,
threadNum:10,
finishCallback:function(responseEntity){
chunkuploader.hideProgressBar();
console.log(responseEntity);
}
}
}
var chunkuploader= $("#chunkupload").chunkUploader(options,);
##例子
var uploadOptions = {
accept:["enc"],
name: "filename",
checkType : "md5",
threadNum:3,
chunkSize: 5 * 1024 * 1024,
maxSize:100*1024*1024*1024,
finishCallback:function (responseEntity) {
if (responseEntity.responseCode == 200) {
$.ajax({
url: '${action_uri}' + '/uploadBackup/' + responseEntity.data + '?_deviceId=${_deviceId}',
timeout: 100000,
async: true,
contentType: 'application/x-www-form-urlencoded',
data: {},
success: function (response) {
var responseEntity = JSON.parse(response);
if (responseEntity.responseCode == 201) {
setTimeout("getUploadStatus('" + responseEntity.data + "')", 2000);
return;
}else{
chunkUploader.hideProgressBar();
$('#uploaderror').show();
$('#uploadsuccess').hide();
$('#uploadErrorinfo').html('${sg:msg("restroe_uploading_failed")}');
uploadBackupinfo = "";
}
},
error: function (data) {
// 去除遮罩
chunkUploader.hideProgressBar();
$('#uploaderror').show();
$('#uploadsuccess').hide();
$('#uploadErrorinfo').html('${sg:msg("restroe_uploading_failed")}');
uploadBackupinfo = "";
}
});
return;
} else {
chunkUploader.hideProgressBar();
$('#uploadsuccess').hide();
$('#uploadErrorinfo').html(responseEntity.message);
$('#uploaderror').show();
}
}
}
chunkUploader =$(".file-upload-div").chunkUploader(uploadOptions);
});
function getUploadStatus(UDID) {
$.ajax({
url: '${action_uri}' + '/getUploadStatus/' + UDID + '?_deviceId=${_deviceId}',
timeout: 100000,
async: true,
contentType: 'application/x-www-form-urlencoded',
data: {},
success: function (response) {
var responseEntity = JSON.parse(response);
if (responseEntity.responseCode == 201) {
setTimeout("getUploadStatus('" + UDID + "')", 2000);
return;
}
if (responseEntity.responseCode == 200) {
chunkUploader.setProgressBar("100%");
$('#uploadsuccess').show();
$('#uploaderror').hide();
uploadBackupinfo = responseEntity.data;
}
else {
$('#uploadErrorinfo').html(responseEntity.message);
$('#uploaderror').show();
$('#uploadsuccess').hide();
uploadBackupinfo = "";
}
chunkUploader.hideProgressBar();
},
error: function (data) {
// 去除遮罩
$('#uploaderror').show();
$('#uploadsuccess').hide();
$('#uploadErrorinfo').html('${sg:msg("restroe_uploading_failed")}');
chunkUploader.hideProgressBar();
uploadBackupinfo = "";
}
});
}
;
</script>
!](https://i-blog.csdnimg.cn/direct/0e12e02c225c4874a4447cc658cbeb2f.png)
注意事项
- 上传前，会检查文件大小和文件的格式是否正常，不正常会直接调用finishCallback
- checkType 是md5时才能做到断点续传，如果fileSize模式只是分片上传
- 上传的文件用完建议自动删除，否则会占用硬盘空间（如果使用公共的我这边会检查超过24小时的会自动清除）
- 分片上传的位置有大小控制，超过100G 会自动删除，所以单次文件上传的大小不能超过100G
- 该组件只是文件上传的组件，具体的业务建议在finishCallback实现
- 默认位置/var///program-data/temp/chunkupload/(chunk或
merge)/[md5或uuid]
参考链接
https://www.cnblogs.com/songsu/p/11798612.html
https://blog.csdn.net/thewindkee/article/details/80189434
https://blog.csdn.net/runner_diego/article/details/51379116
源码
JS
(function ($) {
var _utils = {
mergeUpload: function ($this) {
var path = $this.$fileinput.val();
var fileName = path.substr(path.lastIndexOf('\\') + 1, path.length);
var formData = {};
if ($this.options.checkType == "md5") {
formData.md5 = $this.paramater.md5;
} else {
formData.uuid = $this.paramater.uuid;
formData.fileSize = $this.paramater.fileSize;
}
formData.totalNum = $this.paramater.chunks;
formData.fileName = fileName;
$.ajax({
url: $this.options.mergeUrl,
type: "POST",
async: false,
data: formData,
success: function (response) {
console.log("merge start date:"+new Date().getTime());
console.log(response);
$this.setProgressBar("90%");
var responseEntity = JSON.parse(response);
if (responseEntity.responseCode == 200) {
$this.paramater.mergeThreadId = responseEntity.data;
_utils.mergeStatus($this);
}else{
_utils.finishUpload($this, responseEntity,"mergeUpload create thread is error");
}
},
error: function (data) {
console.log("merge fail");
_utils.finishUpload($this, undefined, $.i18n.map.customer_page_upload_image_fail);
}
});
},
mergeStatus: function ($this) {
console.log("get mergeStatus")
var formData = {};
formData.mergeThreadId = $this.paramater.mergeThreadId;
$.ajax({
url: $this.options.mergeStatus,
type: "POST",
async: false,
data: formData,
success: function (response) {
console.log("get mergeStatus success date:"+new Date().getTime());
console.log(response);
var responseEntity = JSON.parse(response);
if (responseEntity.responseCode == 200) {
if(responseEntity.data == 0){
console.log("continue get mergeStatus");
setTimeout(_utils.mergeStatus, 10000, $this);
}else if(responseEntity.data == 1){
_utils.finishUpload($this, responseEntity);
}else{
responseEntity.responseCode == 500;
_utils.finishUpload($this, responseEntity,"merge fail ,more info server");
}
}else{
responseEntity.message="mergeUpload create thread is error";
}
},
error: function (data) {
console.log("mergeStatus fail");
_utils.finishUpload($this, undefined, $.i18n.map.customer_page_upload_image_fail);
}
});
},
chunkUploadEnd: function ($this, chunkUploadResult) {
if (!$this.paramater.uploading) {
//none
} else if ($this.paramater.uploading && !chunkUploadResult) { //不正常结束
$this.paramater.uploading = false; //提示\
_utils.finishUpload($this, undefined, $.i18n.map.customer_page_upload_image_fail);
} else if ($this.paramater.uploading && chunkUploadResult && $this.paramater.successChunk.length
== $this.paramater.chunks) {//正常结束
$this.paramater.uploading = false; //提示
_utils.mergeUpload($this);
} else {
$this.setProgressBar(
10 + Math.floor($this.paramater.successChunk.length * 75 / $this.paramater.chunks) + "%");
//none
}
},
chunkUpload: function ($this) {
var innnerCurrentChunkNum = $this.options.currentChunkNum;
if ($this.options.threadNum - $this.paramater.currentThreadNum > 0 && innnerCurrentChunkNum < $this.paramater.chunks) {
$this.options.currentChunkNum = $this.options.currentChunkNum + 1;
if ($.inArray(innnerCurrentChunkNum, $this.paramater.successHistoryChunk) > -1) {
$this.paramater.successChunk.push(innnerCurrentChunkNum);
console.log("history exit success:" + innnerCurrentChunkNum);
_utils.chunkUploadEnd($this, true);
} else {
var file = $this.$fileinput.get(0).files[0];
var formData = new FormData();
if ($this.options.checkType == "md5") {
formData.append("md5", $this.paramater.md5);
} else {
formData.append("uuid", $this.options.uuid);
}
formData.append("chunkIndex", innnerCurrentChunkNum);
formData.append("chunkFile", _utils.chunkContent($this, file, innnerCurrentChunkNum));
$.ajax({
url: $this.options.chunkUrl,
timeout: $this.options.threadTimeout,
type: "POST",
async: true,
data: formData,
processData: false,
contentType: false,
success: function (response) {
var responseEntity = JSON.parse(response);
$this.paramater.currentThreadNum = $this.paramater.currentThreadNum - 1;
if (responseEntity.responseCode == 200) {
$this.paramater.successChunk.push(innnerCurrentChunkNum);
// console.log("chunk success date:"+new Date().getTime()+" chunkNum:"+innnerCurrentChunkNum);
_utils.chunkUploadEnd($this, true);
} else {
$this.paramater.failChunk.push(innnerCurrentChunkNum);
console.log("chunkFile fail date:"+new Date().getTime() +" response:"+response);
_utils.chunkUploadEnd($this, false);
}
},
error: function (data) {
$this.paramater.currentThreadNum = $this.paramater.currentThreadNum - 1;
$this.paramater.failChunk.push(innnerCurrentChunkNum);
console.log("fail:" + innnerCurrentChunkNum);
_utils.chunkUploadEnd($this, false);
}
});
$this.paramater.currentThreadNum = $this.paramater.currentThreadNum + 1;
}
console.log("currentThreadNum:" + $this.paramater.currentThreadNum + " currentChunkNum:"
+ $this.options.currentChunkNum);
}
if ($this.paramater.uploading && $this.options.currentChunkNum < $this.paramater.chunks) {
//console.log("keepuploading");
setTimeout(_utils.chunkUpload, 200, $this);
}else{
console.log("thread end currentChunkNum:"+$this.options.currentChunkNum +" uploading:"+$this.paramater.uploading);
}
}, uploadReady: function ($this) {
$this.$chooseButton = $this.$element.find(".uploader-choose");
$this.$uploadButton = $this.$element.find(".uploader-upload");
$this.$chooseButton.addClass('disable');
$this.$uploadButton.val($.i18n.map.crawl_hint_uploading);
var file = $this.$fileinput.get(0).files[0];
$this.paramater.fileSize = file.size;
$this.paramater.chunks = Math.ceil($this.paramater.fileSize / $this.options.chunkSize);
$this.paramater.currentThreadNum = 0;
$this.paramater.failChunk = [];
$this.paramater.successChunk = [];
$this.paramater.successHistoryChunk = [];
$this.paramater.blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
$this.paramater.uploading = true;
$this.paramater.uuid = _utils.randomUuid();
$this.paramater.mergeThreadId="";
$this.options.currentChunkNum = 0;
if ($this.options.progressbar) {
$this.showProgressBar();
}
if ($this.options.checkType == "md5") {
$this.paramater.md5 = "";
var spark = new SparkMD5.ArrayBuffer();
var fileReader = new FileReader();
var tempCurrentChunk = 0;
fileReader.onload = function (e) {
//console.log('read chunk nr', tempCurrentChunk + 1, 'of', $this.paramater.chunks);
spark.append(e.target.result); // Append array buffer
tempCurrentChunk++;
$this.setProgressBar(Math.floor(tempCurrentChunk / $this.paramater.chunks * 9) +1+"%");
if (tempCurrentChunk < $this.paramater.chunks) {
loadNext();
} else {
console.log('finished loading');
$this.paramater.md5 = spark.end();
console.info('computed hash', $this.paramater.md5); // Compute hash
_utils.uploadHistory($this, $this.paramater.md5);
}
};
fileReader.onerror = function () {
console.warn('oops, something went wrong.');
_utils.finishUpload($this, undefined, $.i18n.map.customer_page_upload_image_fail);
};
function loadNext() {
fileReader.readAsArrayBuffer(_utils.chunkContent($this, file, tempCurrentChunk));
}
$this.setProgressBar("1%");
loadNext();
} else {
_utils.createThread($this);
}
},
finishUpload: function ($this, responseEntity, errorMessage) {
console.log("finish upload ")
var temp;
if (responseEntity) {
temp = responseEntity;
if (responseEntity.responseCode == 200) {
if ($this.options.checkType == "md5") {
responseEntity.data = $this.paramater.md5;
} else {
responseEntity.data = $this.paramater.uuid;
}
}
} else {
temp = {"responseCode": 500, "message": errorMessage};
}
$this.$chooseButton.removeClass('disable');
$this.$uploadButton.val( $this.options.uploadBtnText );
if ($this.options.finishCallback) {
$this.options.finishCallback(temp);
} else {
_utils.showInfo($this,temp);
}
},
uploadHistory: function ($this, md5) { //失败不影响上传，只是会重新上传
var fileExist = false;
var responseVal;
$.ajax({
url: $this.options.historyUrl,
type: "GET",
async: false,
data: {"md5": md5},
success: function (response) {
var responseEntity = JSON.parse(response);
if (responseEntity.responseCode == 200) {
if (responseEntity.data == -1) {
console.log("uploadHistory success fileExist");
$this.setProgressBar("95%");
fileExist = true;
responseVal = responseEntity;
} else {
console.log("uploadHistory success");
$this.paramater.successHistoryChunk = responseEntity.data;
}
} else {
console.log("uploadHistory result" + responseEntity.message);
}
},
error: function (data) {
console.log("uploadHistory fail");
}
});
if (fileExist) {
_utils.finishUpload($this, responseVal);
} else {
_utils.createThread($this);
}
},
createThread: function ($this) {
$this.setProgressBar("10%");
console.log("chunks date:"+new Date().getTime()+" chunks:"+$this.paramater.chunks );
for (var i = 0; i < $this.options.threadNum; i++) {
setTimeout(_utils.chunkUpload, 100 * i, $this);
}
},
chunkContent: function ($this, file, chunkNum) {
var start = chunkNum * $this.options.chunkSize,
end = ((start + $this.options.chunkSize) >= $this.paramater.fileSize) ? $this.paramater.fileSize : start + $this.options.chunkSize;
return $this.paramater.blobSlice.call(file, start, end)
}, randomUuid: function () {
function _random() {
return (((1 + Math.random()) * 0x10000) | 0).toString(16).substring(1);
}
return (_random() + _random() + "-" + _random() + "-" + _random() + "-" + _random() + _random()
+ _random());
}, showInfo: function ($this,responseEntity) {
console.error(" chunkUploader not implement finishCallback");
$this.hideProgressBar();
if (responseEntity.responseCode == 200) {
utils.ui.message([Message.ResponseNewMessage(Message.ERRORTYPE.INFO, responseEntity.message)]);
} else {
utils.ui.message([Message.ResponseNewMessage(Message.ERRORTYPE.ERROR, responseEntity.message)]);
}
}
};
var _events = {
onChange: function () {
var file = this.$fileinput.get(0).files[0];
var path = this.$fileinput.val();
var suffix = path.substr(path.lastIndexOf('.') + 1, path.length);
var fileName = path.substr(path.lastIndexOf('\\') + 1, path.length);
var maxSize = this.options.maxSize;
this.messArray = [];
this.paramater.fileSize = file.size;
if (!fileName) {
this.messArray.push($.i18n.map.restore_file_path_type_upload_select);
} else if(this.paramater.fileSize==0 ){
this.messArray.push($.i18n.map.file_size_is_zero);
}else if (this.options.accept && this.options.accept.length!=0&&$.inArray(suffix, this.options.accept) < 0) {
this.messArray.push($.i18n.map.file_type_no_match);
}
if (file && maxSize) {
if (/^[0-9]*[1-9][0-9]*$/.test(maxSize)) {//纯数字
if (this.paramater.fileSize > parseFloat(maxSize)) {
this.messArray.push($.i18n.map.base_ret_status_file_exceeds_max_limit);
}
} else {//带单位
//单位转换
var numPart = parseFloat(maxSize.substr(0, maxSize.length - 2));
var strPart = maxSize.substr(maxSize.length - 2, maxSize.length);
switch (strPart) {
case 'KB':
numPart = numPart * 1024;
break;
case 'MB':
numPart = numPart * 1048576;
break;
case 'GB':
numPart = numPart * 1073741824;
break;
case 'TB':
numPart = numPart * 1099511627776;
break;
default:
break;
}
if (numPart && file.size > numPart) {
this.messArray.push($.i18n.map.base_ret_status_file_exceeds_max_limit);
}
}
}
this.$filename.val(fileName);
if (!this.options.showUploadBtn && this.options.autoUpload) {
this.upload();
}
},
onBtnClick: function () {
if (!this.$fileinput.hasClass('disable')) {
this.$fileinput.trigger('click')
}
},
mouseover: function (e) {
if (!this.$fileinput.hasClass('disable')) {
if (this.$fileinput.val() || this.$filename.val()) {
this.$clear.show();
} else {
this.$clear.hide();
}
}
},
mouseout: function () {
if (!this.$fileinput.hasClass('disable')) {
this.$clear.hide()
}
},
clear: function () {
this.$fileinput.val('');
this.$filename.val('');
this.messArray = [];
this.messArray.push($.i18n.map.restore_file_path_type_upload_select);
}
};
var html =
'<label class="js-hover calendar-selection-l inline-block">' +
'<div class="inline-block fileup-title"></div>' +
'<input type="text" readonly="readonly">' +
'<a href="javascript:;" class="i-tipsError js-clear"></a>' +
'</label>' +
'<a href="javascript:;" class="btn btn-gray uploader-choose margin-l-s">' + $.i18n.map.button_select
+ '</a>' +
'<input type="file" name="" class="" accept="" style="display: none">';
var progress =
'<div class="mod-popup-mask upload-progress" style="display:none">'
+ ' <div class="mod-popup-progress">'
+ ' <p>' + $.i18n.map.crawl_hint_uploading + '</p>'
+ ' <div class="mod-progress-bar">'
+ ' <div class="progress-bar" style="width: 0%;"></div> <span class="progress-number">0%</span>'
+ ' </div>'
+ ' </div>'
+ ' </div>';
var defaultSetting = {
accept:[],
maxSize: 1 * 1024 * 1024 * 1024,
chunkSize: 2 * 1024 * 1024,
threadNum: 5,
threadTimeout: 60000,
inputFileName: "uploadFile",
placeholder: $.i18n.map.source_dest_common_choose_file_blank,
showUploadBtn: true,
uploadBtnText: $.i18n.map.common_upload,
autoUpload: true,
historyUrl: root_path + "/util/upload/history",
chunkUrl: root_path + "/util/upload/chunkUpload",
mergeUrl: root_path + "/util/upload/mergeFile",
mergeStatus: root_path + "/util/upload/mergeStatus",
progressbar: true,//进度条,校验10%、分片上传75%、合成10%、最后的5%回掉关闭
checkType: "md5", //fileSize
finishCallback: undefined
};
var paramater = {
blobSlice:undefined,
//公共参数
uploading: false,
currentThreadNum: 0,
chunks: 0, //总块数
md5: "",
fileSize: "",
uuid: "",
md5Array: [], //废弃
currentChunkNum: 0,
failChunk: [], //后期扩展
successChunk: [], //用来判断上传成功结束，多线程不能根据最后一块成功判断成功
successHistoryChunk: [] //用来判断上传成功结束，多线程不能根据最后一块成功判断成功
}
var chunkUploader = function ($element, options) {
if ($element.children().length < 1) {
$element.addClass('inline-block');
$element.append(html);
if($(document).find(".main-wrap").length!=0){
this.$progressElement = $(document).find(".main-wrap");
}else{
this.$progressElement = $( "body" );
}
this.$progressElement.append(progress);
if (options.showUploadBtn) {
$element.append('<input type="button" class="btn btn-blue margin-l-s uploader-upload" value="'
+ options.uploadBtnText + '">');
}
}
this.options = options;
this.paramater = paramater;
this.$element = $element;
this.$fileinput = $element.find("input[type='file']");
this.$filename = $element.find("input[type='text']");
this.$hoverarea = $element.find(".js-hover");
this.$clear = $element.find(".js-clear");
this.$chooseButton = $element.find(".uploader-choose");
this.$uploadButton = $element.find(".uploader-upload");
if (options.accept) {
this.$fileinput.prop('accept', options.accept.map(function (item) {
return '.' + item
}).join(','));
}
this.$fileinput.prop('name', options.inputFileName);
this.$fileinput.prop('id', options.inputFileName);
this.$filename.prop('placeholder', options.placeholder);
this.messArray = [];
//绑定事件
this.$fileinput.off("change.sux.chunkUploader.change")
.on("change.sux.chunkUploader.change", $.proxy(_events.onChange, this));
this.$chooseButton.off("click.sux.chunkUploader.click")
.on("click.sux.chunkUploader.click", $.proxy(_events.onBtnClick, this));
this.$hoverarea.off("mouseenter.sux.chunkUploader.mouseenter")
.on("mouseenter.sux.chunkUploader.mouseenter", $.proxy(_events.mouseover, this));
this.$hoverarea.off("mouseleave.sux.chunkUploader.mouseleave")
.on("mouseleave.sux.chunkUploader.mouseleave", $.proxy(_events.mouseout, this));
this.$clear.off("click.sux.chunkUploader.click")
.on("click.sux.chunkUploader.click", $.proxy(_events.clear, this));
this.$uploadButton.off("click.sux.chunkUploader.click")
.on("click.sux.chunkUploader.click", $.proxy(this.upload, this));
this.messArray.push($.i18n.map.restore_file_path_type_upload_select);
};
chunkUploader.prototype.getFile = function () {
return this.$fileinput.get(0).files[0];
};
chunkUploader.prototype.showProgressBar = function () {
this.$progressElement.find(".upload-progress").show();
this.$progressElement.find(".progress-bar").width("0%");
this.$progressElement.find(".progress-number").html("0%");
};
chunkUploader.prototype.hideProgressBar = function () {
this.$progressElement.find(".upload-progress").hide()
};
chunkUploader.prototype.setProgressBar = function (percent) {
this.$progressElement.find(".progress-bar").width(percent);
this.$progressElement.find(".progress-number").html(percent);
};
chunkUploader.prototype.getFileName = function () {
return this.$filename.val();
};
chunkUploader.prototype.upload = function () {
//progressbar
if (this.messArray.length > 0) {
_utils.finishUpload(this, undefined, this.messArray[0]);
return;
}
_utils.uploadReady(this);
};
$.fn.chunkUploader = function (options, finishCallback) { //need finishCallback
if (!options) {
options = {};
}
if (!$(this).data('sux.chunkUploader')) {
/* if (finishCallback) {
options.finishCallback = finishCallback;
}*/
var instance = new chunkUploader($(this), $.extend({}, defaultSetting, options));
$(this).data('sux.chunkUploader', instance);
return instance;
} else {
return $(this).data('sux.chunkUploader');
}
}
}(jQuery));
java
@Controller
@RequestMapping("/util/upload")
public class ChunkUploadController extends BaseController {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private Long MAX_SPACE = Integer.valueOf(BaseConfig.getItem("chunk_upload_place_max", "100")) * 1024 * 1024 * 1024L;
private FileAlterationMonitor fileAlterationMonitor = null;
private ConcurrentHashMap mergeHashMap = new ConcurrentHashMap<>(); //0 正在合并 -1合并失败 1 合并成功
//根据MD5找到之前的上传记录
@RequestMapping(value = "/history", method = RequestMethod.GET)
@ResponseBody
public ResponseEntity history(HttpServletRequest request, HttpServletResponse response) {
logger.debug("history access");
try {
initialize();
String md5 = request.getParameter("md5");
String chunkPath = ChunkFileUtil.getChunkFilePath(md5);
String mergeFolderPath = ChunkFileUtil.getMergeFolderPath(md5);
File folder = new File(mergeFolderPath);
if (folder.exists() && folder.listFiles() != null && folder.listFiles().length == 0) {
folder.delete();
logger.info("history exists but is empty");
}
if (folder.exists()) {
logger.info("history exists not upload");
return new ResponseEntity(ResponseStatus.SUCCESS, -1);
} else {
List<Integer> history = ChunkFileUtil.getChunkUploadHistory(chunkPath);
return new ResponseEntity(ResponseStatus.SUCCESS, history);
}
} catch (Exception e) {
logger.error("history error e:", e);
return new ResponseEntity<>(ResponseStatus.INTERNAL_ERROR);
}
}
@RequestMapping(value = "/chunkUpload", method = RequestMethod.POST)
@ResponseBody
public ResponseEntity<Map<String, Object>> chunkUpload(HttpServletRequest request, HttpServletResponse response) {
logger.debug("chunkUpload access");
boolean result;
try {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
String md5 = request.getParameter("md5"); //filesize 模式没有uuid
String uuid = request.getParameter("uuid");
int chunkIndex = Integer.valueOf(request.getParameter("chunkIndex"));
MultipartFile configFile = multipartRequest.getFile("chunkFile");
String chunkPath = StringUtils.isEmpty(md5) ? ChunkFileUtil.getChunkFilePath(uuid) : ChunkFileUtil.getChunkFilePath(md5);
InputStream inputStream = configFile.getInputStream();
result = ChunkFileUtil.chunkUpload(chunkPath, chunkIndex, inputStream);
} catch (Exception e) {
result = false;
logger.error("chunkUpload error e:", e);
}
return result ? new ResponseEntity<>(ResponseStatus.SUCCESS) : new ResponseEntity<>(ResponseStatus.INTERNAL_ERROR);
}
@RequestMapping(value = "/mergeFile", method = RequestMethod.POST)
@ResponseBody
public ResponseEntity mergeFile(HttpServletRequest request, HttpServletResponse response) {
logger.info("mergeFile access");
String threadId = DateFormatUtils.format(new Date(), "yyyyMMdd'T'HHmmssSSS");
//MD5
String md5 = request.getParameter("md5");
//fileSize
String uuid = request.getParameter("uuid");
String fileSizeStr = request.getParameter("fileSize");
try {
int totalNum = Integer.valueOf(request.getParameter("totalNum"));
String fileName = request.getParameter("fileName");
String chunkFolderPath = StringUtils.isEmpty(md5) ? ChunkFileUtil.getChunkFilePath(uuid) : ChunkFileUtil.getChunkFilePath(md5);
String mergeFolderPath = StringUtils.isEmpty(md5) ? ChunkFileUtil.getMergeFolderPath(uuid) : ChunkFileUtil.getMergeFolderPath(md5);
mergeHashMap.put(threadId,0);
Thread thread = new Thread(() -> {
boolean result;
if (StringUtils.isEmpty(md5)) {
result = ChunkFileUtil.mergeChunk(Long.valueOf(fileSizeStr), totalNum, fileName, chunkFolderPath, mergeFolderPath);
} else {
result = ChunkFileUtil.mergeChunk(md5, totalNum, fileName, chunkFolderPath, mergeFolderPath);
}
logger.info("mergeFiles threadId : {} result:{}", threadId,result);
mergeHashMap.put(threadId,result?1:-1);
});
thread.start();
} catch (Exception e) {
logger.error("mergeFiles error e:", e);
return new ResponseEntity(ResponseStatus.INTERNAL_ERROR);
}
logger.info("mergeFiles threadId:{} ", threadId);
return new ResponseEntity(ResponseStatus.SUCCESS,threadId);
}
@RequestMapping(value = "/mergeStatus", method = RequestMethod.POST)
@ResponseBody
public ResponseEntity mergeStatus(HttpServletRequest request, HttpServletResponse response) {
logger.info("mergeStatus access");
String threadId = request.getParameter("mergeThreadId");
logger.info("mergeStatus threadId:{}",threadId);
return new ResponseEntity(ResponseStatus.SUCCESS,mergeHashMap.containsKey(threadId)?mergeHashMap.get(threadId):-1);
}
//用到之后增加定时检查，定时扫描任务防止上传空间过大，只监控文件夹新增
private void initialize() throws Exception {
logger.debug("task initialize");
if (fileAlterationMonitor != null) {
return;
}
FileAlterationObserver fileAlterationObserver = new FileAlterationObserver(ChunkFileUtil.getUploadPath(),
FileFilterUtils.and(new IOFileFilter() {
@Override
public boolean accept(File file) {
return file.isDirectory();
}
@Override
public boolean accept(File dir, String name) {
System.out.println("accept:" + dir.getName() + " name:" + name);
return false;
}
}));
fileAlterationObserver.addListener(new FileAlterationListener() {
@Override
public void onStart(FileAlterationObserver observer) {
//none
}
@Override
public void onDirectoryCreate(File directory) {
executeDelete();
}
@Override
public void onDirectoryChange(File directory) {
//none
}
@Override
public void onDirectoryDelete(File directory) {
//none
}
@Override
public void onFileCreate(File file) {
//none
}
@Override
public void onFileChange(File file) {
//none
}
@Override
public void onFileDelete(File file) {
//none
}
@Override
public void onStop(FileAlterationObserver observer) {
//none
}
});
fileAlterationMonitor = new FileAlterationMonitor(TimeUnit.SECONDS.toMillis(60), fileAlterationObserver); //TimeUnit.MINUTES.toMillis(1)
fileAlterationMonitor.setThreadFactory(new ThreadFactoryBuilder().setNameFormat("chunk upload monitor")
.setPriority(Thread.MAX_PRIORITY).build());
fileAlterationMonitor.start();
}
@PreDestroy
private void terminate() throws Exception {
if (fileAlterationMonitor != null) {
fileAlterationMonitor.stop();
}
}
private synchronized void executeDelete() {
//检查文件夹的大小大于200G
File chunkUploadDir = new File(ChunkFileUtil.getUploadPath());
if (chunkUploadDir.exists() && chunkUploadDir.isDirectory()) {
if (MAX_SPACE < FileUtil.sizeOfDirectory(chunkUploadDir)) {
File chunkDir = new File(ChunkFileUtil.getChunkRootPath());
File mergeDir = new File(ChunkFileUtil.getMergeRootPath());
File[] chunkFileArray = chunkDir.listFiles();
File[] mergeFileArray = mergeDir.listFiles();
List<File> list = new ArrayList<>();
if (chunkFileArray != null) {
list.addAll(Arrays.asList(chunkFileArray));
}
if (mergeFileArray != null) {
list.addAll(Arrays.asList(mergeFileArray));
}
list.sort((f1, f2) -> f1.lastModified() > f2.lastModified() ? 1 : -1);
do {
if (list.size() > 0) {
File temp = list.get(0);
list.remove(0);
try {
FileUtil.deleteFolder(temp.getAbsoluteFile().getPath());
} catch (Exception e) {
logger.error("executeDelete error e:", e);
}
}
} while (MAX_SPACE < FileUtil.sizeOfDirectory(chunkUploadDir) && list.size() > 0);
}
}
}
}
public class ChunkFileUtil extends FileUtils {
private static final Logger logger = LoggerFactory.getLogger(ChunkFileUtil.class);
private static String chunk_upload_path;
static {
try {
chunk_upload_path = BaseConfig.getItem("chunk_upload_path", "tets");
} catch (Exception e) {
logger.error("get BaseConfig chunk_upload_path error", e);
}
}
public static boolean chunkUpload(String chunkFolderPath, int chunkIndex, InputStream inputStream) {
boolean flag = true;//标示是否成功
String messageInfo = "success";
File thunkFile;
String chunkPath = chunkFolderPath;
String chunkFilePath = chunkPath + chunkIndex;
thunkFile = new File(chunkFilePath);
try {
FileUtil.createNotExistPath(chunkPath);
if (thunkFile.exists()) {
thunkFile.delete();
thunkFile = new File(chunkFilePath); //TODO test delete do not create
logger.info("chunkUpload old exit but md5 not match will delete old file"); //TODO
}
try (FileOutputStream fos = new FileOutputStream(thunkFile)) {
byte buffer[] = new byte[1024];
int length = 0;
while ((length = inputStream.read(buffer)) != -1) {
fos.write(buffer, 0, length);
}
} catch (Exception e) {
thunkFile.delete();
messageInfo = "storing local exceptions ";
logger.error(messageInfo, e);
flag = false;
}
logger.info("chunkUpload end " + messageInfo); //TODO
} catch (Exception e) {
logger.error("chunkUpload error e:", e);
flag = false;
} finally {
if (!flag) {
thunkFile.delete();
}
}
return flag;
}
public static List<Integer> getChunkUploadHistory(String chunkFolderPath) {
File tempFileFolder = ChunkFileUtil.getFile(chunkFolderPath);
List<Integer> history = new ArrayList<>();
if (tempFileFolder.exists()) {
File[] files = tempFileFolder.listFiles();
if (files != null) {
for (File f : files) {
history.add(Integer.valueOf(f.getName()));
}
}
}
return history;
}
public static boolean mergeChunk(String md5, int totalNum, String fileName, String chunkFolderPath, String mergeFolderPath) {
boolean flag = mergeChunk(totalNum, fileName, chunkFolderPath, mergeFolderPath, () -> {
String tempMD5 = MD5Util.md5sumForFile(mergeFolderPath + fileName);
logger.info("mergeFiles check file md5 {} merge after {}", md5, tempMD5); //TODO
return tempMD5.equals(md5);
});
return flag;
}
public static boolean mergeChunk(Long fileSize, int totalNum, String fileName, String chunkFolderPath, String mergeFolderPath) {
boolean flag = mergeChunk(totalNum, fileName, chunkFolderPath, mergeFolderPath, () -> {
File targetFile = new File(mergeFolderPath + fileName);
logger.info("mergeFiles check file size {} merge after {}", fileSize, targetFile.length()); //TODO
return targetFile.length() == fileSize;
});
if (!flag) { //uuid这种删除失败之后清空分片，分片保留无意义
try {
FileUtil.deleteFolder(chunkFolderPath);
} catch (Exception e) {
logger.error("mergeChunk fail delete chunk Folder e:", e);
}
}
return flag;
}
/**
* 合并分片文件
*/
public static boolean mergeChunk(int totalNum, String fileName, String chunkFolderPath, String mergeFolderPath, Supplier<Boolean> checkFile) {
String messageInfo = "success";
boolean flag = true;
File chunkFolder = FileUtil.getFile(chunkFolderPath);
try {
File[] files = chunkFolder.listFiles();
if (files != null && files.length >= totalNum) {
File[] childrens = new File[totalNum];
//检查名称
for (int i = 0; i < totalNum; i++) {
childrens[i] = new File(chunkFolderPath + i);
if (!childrens[i].exists() || !childrens[i].isFile()) {
flag = false;
messageInfo = (chunkFolderPath + i) + " no exist or not file";
logger.info(messageInfo);
break;
}
}
if (flag) {
try {
FileUtil.deleteFolder(mergeFolderPath);
FileUtil.createNotExistPath(mergeFolderPath);
File targetFile = new File(mergeFolderPath + fileName);
int bufSize = 1024;
try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(targetFile))) {
byte[] buffer = new byte[bufSize];
for (int i = 0; i < childrens.length; i++) {
try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(childrens[i]))) {
int readCount;
while ((readCount = inputStream.read(buffer)) > 0) {
outputStream.write(buffer, 0, readCount);
}
}
}
}
flag = true;
} catch (Exception e) {
messageInfo = "mergeFiles merge error,mergeFiles " + mergeFolderPath;
logger.error(messageInfo, e);
flag = false;
}
} else {
//none
}
} else {
flag = false;
messageInfo = "mergeFiles totalNum error,totalNum: " + totalNum + " chunkFileNum:" + files.length;
logger.info(messageInfo);
}
if (flag && checkFile != null) {
flag = checkFile.get();
messageInfo = "mergeFiles check result:" + flag;
logger.info(messageInfo);
}
} catch (Exception e) {
messageInfo = "mergeFiles merge error e:" + e.getMessage();
logger.error(messageInfo, e);
} finally {
if (flag) {
try {
FileUtil.deleteFolder(chunkFolderPath);
} catch (Exception e) {
logger.error("mergeChunk success delete chunk Folder e:", e);
}
}
if (!flag) {
try {
FileUtil.deleteFolder(mergeFolderPath);
} catch (Exception e) {
logger.error("mergeChunk fail delete mergeFolder e:", e);
}
}
}
logger.info("mergeFiles end {} {}", mergeFolderPath, messageInfo); //TODO
return flag;
}
public static String getUploadPath() {
return chunk_upload_path;
}
public static String getChunkRootPath() {
return chunk_upload_path + "chunk/";
}
public static String getMergeRootPath() {
return chunk_upload_path + "merge/";
}
public static String getChunkFilePath(String key) {
return getChunkRootPath() + key + "/";
}
public static String getMergeFolderPath(String key) {
return getMergeRootPath() + key + "/";
}
/**
* 根据key 获取上传文件的地址
*
* @param key(uuid or md5)
*/
public static String getUploadFilePath(String key) {
String mergeFilePath = getMergeFolderPath(key);
File folder = new File(mergeFilePath);
if (folder.exists() && folder.listFiles() != null && folder.listFiles().length > 0) {
return mergeFilePath + folder.listFiles()[0].getName();
}
return null;
}
}
public static String md5sumForFile(String inputpath) {
Preconditions.checkNotNull(inputpath, "input path is null!");
String result = null;
FileInputStream fileInputStream = null;
try {
fileInputStream = new FileInputStream(Paths.get(inputpath).toString());
result = DigestUtils.md5Hex(fileInputStream);
} catch (FileNotFoundException e) {
logger.error("file not found", e);
} catch (IOException e) {
logger.error("io exception", e);
} finally {
if (fileInputStream != null) {
try {
fileInputStream.close();
} catch (IOException e) {
logger.error("close file input stream error", e);
}
}
}
return result;
}
扫一扫
1471
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
