1.之前有个需求要上传一批700w词导入到线上的kafka,拼接好json后发现文件有2G多,而且线上web只支持传线文件只有30m以内,然后就写了这个分片上传。
分片上传思想:js将文件切割每10M(在js处可以调整大小)一个,循环调用后台接口(这里目前是单线程,可以改为多线程),上传至最后一个文件时进行文 件合并操作。
注意:如果是集群服务则需要将文件上传至文件服务器,目前代码是将文件上传至本机进行合并,集群则会导致文件不会落在一台机器上,导致合并时找不到文件而报错。
后端java代码
package com.yicong.boke.test;
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;
/**
* @Description:
* @Author yicong
* @Date 2020年4月2日
*/
@Controller
@SuppressWarnings("all")
public class DataManagerController {
public static final Logger LOGGER = Logger.getLogger(DataManagerController.class);
private Map<String, String> maps = new ConcurrentHashMap<String, String>();
@PostMapping("v1/dataManager/uploadKafkaTask")
@ResponseBody
public Result uploadTask(FileFragmentation fileFragmentation, HttpServletRequest request) {
Long fileIndex = fileFragmentation.getFileIndex();
Long fileTotal = fileFragmentation.getFileTotal();
Result result = createTempFile(fileFragmentation);
if (result.getCode() != 200) {
return result;
}
if (fileIndex < fileTotal) {
return new Result(200, "上传中", 1);
}
result = mergeFile(fileFragmentation);
if (result.getCode() != 200) {
Object data = result.getData();
if (data != null) {
File mergeFile = (File) data;
FileUtils.deleteQuietly(mergeFile);
}
return result;
} else {
File mergeFile = (File) result.getData();
result = mergeFileToKafka(mergeFile);
FileUtils.deleteQuietly(mergeFile);
return result;
}
}
private Result mergeFileToKafka(File mergeFile) {
long mergeFileStartTime = System.currentTimeMillis();
long mergeFileCost = System.currentTimeMillis() - mergeFileStartTime;
LOGGER.info("DataManagerController mergeFile 文件合并成功,耗时:" + mergeFileCost + "ms,即将执行校验。。。");
return new Result(200, "文件合并完成。。。", 2);
}
private Result createTempFile(FileFragmentation fileFragmentation) {
MultipartFile fileData = fileFragmentation.getFileData();
String tempMapFileName = getTempFileName(fileFragmentation.getFileMd5(), fileFragmentation.getFileIndex(),
fileFragmentation.getFileName());
File tempFile = null;
try {
byte[] fileBytes = fileData.getBytes();
tempFile = File.createTempFile(tempMapFileName, null);
FileUtils.writeByteArrayToFile(tempFile, fileBytes);
maps.put(tempMapFileName, tempFile.getAbsolutePath());
} catch (Exception e) {
FileUtils.deleteQuietly(tempFile);
LOGGER.error("DataManagerController createTempFile 临时文件创建失败 e=" + e.getMessage());
return new Result(400, "临时文件创建失败。。。", null);
}
return new Result(200, "临时文件创建成功。。。", null);
}
private static String getTempFileName(String md5, Long index, String fileName) {
return md5 + "_" + index + "_" + fileName;
}
private Result mergeFile(FileFragmentation fileFragmentation) {
String fileMd5 = fileFragmentation.getFileMd5();
String fileName = fileFragmentation.getFileName();
Long fileTotal = fileFragmentation.getFileTotal();
File mergeFile = null;
try {
mergeFile = File.createTempFile(fileMd5, fileFragmentation.getFileName());
} catch (IOException e) {
LOGGER.error("DataManagerController mergeFile 临时文件创建失败 e=" + e.getMessage());
return new Result(400, "临时文件创建失败。。。", mergeFile);
}
for (long i = 1; i <= fileTotal; i++) {
String tempFileName = getTempFileName(fileMd5, i, fileName);
String tempFilePath = maps.get(tempFileName);
File tempFile = new File(tempFilePath);
try {
byte[] data = FileUtils.readFileToByteArray(tempFile);
// 文件合并
FileUtils.writeByteArrayToFile(mergeFile, data, true);
} catch (Exception e) {
// 合并文件失败
LOGGER.error("DataManagerController mergeFile 文件合并出错 e=" + e.getMessage());
} finally {
// 删除文件
FileUtils.deleteQuietly(tempFile);
maps.remove(tempFileName);
}
}
return new Result(200, "文件合并完成。。。", mergeFile);
}
public class Result {
/* 状态码 */
private Integer code;
/* 提示信息 */
private String message;
/* 具体的内容 */
private Object data;
public Result(Integer code, String message, Object data) {
super();
this.code = code;
this.message = message;
this.data = data;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public Integer getCode() {
return code;
}
public void setCode(Integer code) {
this.code = code;
}
public Object getData() {
return data;
}
public void setData(Object data) {
this.data = data;
}
public Result() {
super();
}
}
/**
* <Description> 文件分片上传实体类,和前端代码相对应<br>
*
* @Author yicong
* @Date 2020年4月21日
*
*/
public class FileFragmentation {
/**
* 文件数据
*/
private MultipartFile fileData;
/**
* 文件名称
*/
private String fileName;
/**
* 文件大小
*/
private Long fileSize;
/**
* 文件MD5
*/
private String fileMd5;
/**
* 文件总个数
*/
private Long fileTotal;
/**
* 文件当前分片数
*/
private Long fileIndex;
public MultipartFile getFileData() {
return fileData;
}
public void setFileData(MultipartFile fileData) {
this.fileData = fileData;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public Long getFileSize() {
return fileSize;
}
public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}
public String getFileMd5() {
return fileMd5;
}
public void setFileMd5(String fileMd5) {
this.fileMd5 = fileMd5;
}
public Long getFileTotal() {
return fileTotal;
}
public void setFileTotal(Long fileTotal) {
this.fileTotal = fileTotal;
}
public Long getFileIndex() {
return fileIndex;
}
public void setFileIndex(Long fileIndex) {
this.fileIndex = fileIndex;
}
}
}
引用的js文件
/*
* 文件分段上传jquery插件
* 分离了原来的进度动画,现在用户可以自定义自己的动画和按钮,分别提供了各种回调事件以便处理
*/
var fcup_upload = {
fcup: function (config) {
jQuery.extend(config);
if (jQuery.upId && jQuery.upUrl) {
jQuery.domHtml = jQuery('#' + jQuery.upId).html();
jQuery.fcup_addFileInput();
}
},
fcup_limitFileSize: function (file, limitSize) {
var arr = ["KB", "MB", "GB"],
limit = limitSize.toUpperCase(),//toUpperCase方法用于把字符串转换为大写。
limitNum = 0;
for (var i = 0; i < arr.length; i++) {
var leval = limit.indexOf(arr[i]);
if (leval > -1) {
limitNum = parseInt(limit.substr(0, leval)) * Math.pow(1024, (i + 1));
break;
}
}
if (file.size > limitNum) {
return false;
}
return true;
},
upStop: function (err) {
jQuery.upError = err;
},
upStatus: function () {
if (jQuery.upError) {
if (typeof jQuery.upStop == 'function') {
jQuery.upStop(jQuery.upError);
}
return false;
}
return true;
},
fcup_getPercent: function (num, total) {
num = parseFloat(num);
total = parseFloat(total);
if (isNaN(num) || isNaN(total)) {
return "-";
}
return total <= 0 ? 0 : (Math.round(num / total * 10000) / 100.00);
},
fcup_addFileInput: function () {
jQuery.upInputId = jQuery.upId + '_input';
var C = jQuery('#' + jQuery.upId).attr("class");
var X = jQuery('#' + jQuery.upId).position().top;
var Y = jQuery('#' + jQuery.upId).position().left;
var W = jQuery('#' + jQuery.upId).innerWidth();
var H = jQuery('#' + jQuery.upId).innerHeight();
var html = jQuery.domHtml;
if (C) {
html += '<input type="file" id="' + jQuery.upInputId + '" class="' + C + '" οnchange="jQuery.fcup_upload()" style="position:absolute;left:' + Y + 'px;top:' + X + 'px;opacity:0;z-index:9999;width:' + W + 'px;height:' + H + 'px;">';
} else {
html += '<input type="file" id="' + jQuery.upInputId + '" οnchange="jQuery.fcup_upload()" style="position:absolute;left:' + Y + 'px;top:' + X + 'px;opacity:0;z-index:9999;width:' + W + 'px;height:' + H + 'px;">';
}
jQuery('#' + jQuery.upId).html(html);
},
fcupUpFileInput: function () {
var X = jQuery('#' + jQuery.upId).position().top;
var Y = jQuery('#' + jQuery.upId).position().left;
var W = jQuery('#' + jQuery.upId).innerWidth();
var H = jQuery('#' + jQuery.upId).innerHeight();
var obj = document.getElementById(jQuery.upInputId);
obj.style.cssText = 'position:absolute;left:' + Y + 'px;top:' + X + 'px;opacity:0;z-index:9999;width:' + W + 'px;height:' + H + 'px;';
},
fcup_upload: function () {
var result = '';
jQuery.upError = '';
var file = jQuery('#' + jQuery.upInputId)[0].files[0];
if (!file) {
return false;
}
if (typeof jQuery.upStart == 'function') {
jQuery.upStart();
}
name = file.name;
size = file.size;
index1 = name.lastIndexOf(".");
if (!jQuery.upShardSize) {
jQuery.upShardSize = 2;
}
var fileMD5 = jQuery.getFileMd5(file);
var index2 = name.length,
suffix = name.substring(index1 + 1, index2),
shardSize = jQuery.upShardSize * 1024 * 1024,
succeed = 0,
shardCount = Math.ceil(size / shardSize);
if (jQuery.upType) {
uptype = jQuery.upType.split(",");
if (jQuery.inArray(suffix, uptype) == -1) {
jQuery.upError = '不允许上传的文件类型-' + suffix;
}
}
if (jQuery.upMaxSize) {
if (!jQuery.fcup_limitFileSize(file, jQuery.upMaxSize + 'MB')) {
jQuery.upError = '上传文件过大';
}
}
if (jQuery.upStatus() == false) {
return false;
}
setInterval("jQuery.fcupUpFileInput();",500);
var re = [];
var start, end = 0;
for (var i = 0; i < shardCount; ++i) {
re[i] = [];
start = i * shardSize,
end = Math.min(size, start + shardSize);
re[i]["fileData"] = file.slice(start, end);
re[i]["fileName"] = name;
re[i]["fileSize"] = size; //文件大小
}
const URL = jQuery.upUrl;
var i2 = 0, i3 = 1, fcs = Array();
var xhr = new XMLHttpRequest();
function ajaxStack(stack) {
if (jQuery.upStatus() == false) {
return;
}
var form = new FormData();
if (stack[i2]) {
fcs = stack[i2];
form.append("fileData", fcs['fileData']);
form.append("fileName", fcs['fileName']);
form.append("fileSize", fcs['fileSize']);
form.append("fileMd5", fileMD5);
form.append("fileTotal", shardCount);
form.append("fileIndex", i3);
xhr.open('POST', URL, true);
xhr.onload = function () {
ajaxStack(stack);
}
xhr.onreadystatechange = function () {
if (jQuery.upStatus() == false) {
return;
}
if (xhr.readyState == 4 && xhr.status == 200) {
var data = xhr.responseText ? eval('(' + xhr.responseText + ')') : '';
++succeed;
var cent = jQuery.fcup_getPercent(succeed, shardCount);
if (typeof jQuery.upEvent == 'function') {
jQuery.upEvent(cent);
}
if (cent == 100) {
setTimeout(function () {
if (typeof jQuery.upCallBack == 'function') {
jQuery.upCallBack(data);
}
}, 500);
} else {
if (typeof jQuery.upCallBack == 'function') {
jQuery.upCallBack(data);
}
}
}
}
xhr.send(form);
i2++; i3++;
form.delete('fileData');
form.delete('fileName');
form.delete('fileSize');
form.delete('fileMd5');
form.delete('fileTotal');
form.delete('fileIndex');
}
}
ajaxStack(re);
re = null,
file = null;
},
getFileMd5: function (file) {
var blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
chunkSize = jQuery.upShardSize,
chunks = Math.ceil(file.size / chunkSize),
currentChunk = 0,
spark = new SparkMD5.ArrayBuffer(),
frOnload = function (e) {
spark.append(e.target.result); // append array buffer
currentChunk++;
if (currentChunk < chunks) loadNext();
},
frOnerror = function () { };
function loadNext() {
var fileReader = new FileReader();
fileReader.onload = frOnload;
fileReader.onerror = frOnerror;
var start = currentChunk * chunkSize,
end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;
fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
};
loadNext();
return spark.end();
}
};
(function (jQuery) {
jQuery.extend(fcup_upload);
})(jQuery);
/*
* spark-md5文件 用于计算文件md5值,这是一个外部类
*/
(function (factory) { if (typeof exports === "object") { module.exports = factory() } else if (typeof define === "function" && define.amd) { define(factory) } else { var glob; try { glob = window } catch (e) { glob = self } glob.SparkMD5 = factory() } })(function (undefined) { "use strict"; var add32 = function (a, b) { return a + b & 4294967295 }, hex_chr = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"]; function cmn(q, a, b, x, s, t) { a = add32(add32(a, q), add32(x, t)); return add32(a << s | a >>> 32 - s, b) } function md5cycle(x, k) { var a = x[0], b = x[1], c = x[2], d = x[3]; a += (b & c | ~b & d) + k[0] - 680876936 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[1] - 389564586 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[2] + 606105819 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[3] - 1044525330 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[4] - 176418897 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[5] + 1200080426 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[6] - 1473231341 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[7] - 45705983 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[8] + 1770035416 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[9] - 1958414417 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[10] - 42063 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[11] - 1990404162 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & c | ~b & d) + k[12] + 1804603682 | 0; a = (a << 7 | a >>> 25) + b | 0; d += (a & b | ~a & c) + k[13] - 40341101 | 0; d = (d << 12 | d >>> 20) + a | 0; c += (d & a | ~d & b) + k[14] - 1502002290 | 0; c = (c << 17 | c >>> 15) + d | 0; b += (c & d | ~c & a) + k[15] + 1236535329 | 0; b = (b << 22 | b >>> 10) + c | 0; a += (b & d | c & ~d) + k[1] - 165796510 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[6] - 1069501632 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[11] + 643717713 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[0] - 373897302 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[5] - 701558691 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[10] + 38016083 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[15] - 660478335 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[4] - 405537848 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[9] + 568446438 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[14] - 1019803690 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[3] - 187363961 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[8] + 1163531501 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b & d | c & ~d) + k[13] - 1444681467 | 0; a = (a << 5 | a >>> 27) + b | 0; d += (a & c | b & ~c) + k[2] - 51403784 | 0; d = (d << 9 | d >>> 23) + a | 0; c += (d & b | a & ~b) + k[7] + 1735328473 | 0; c = (c << 14 | c >>> 18) + d | 0; b += (c & a | d & ~a) + k[12] - 1926607734 | 0; b = (b << 20 | b >>> 12) + c | 0; a += (b ^ c ^ d) + k[5] - 378558 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[8] - 2022574463 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[11] + 1839030562 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[14] - 35309556 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[1] - 1530992060 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[4] + 1272893353 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[7] - 155497632 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[10] - 1094730640 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[13] + 681279174 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[0] - 358537222 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[3] - 722521979 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[6] + 76029189 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (b ^ c ^ d) + k[9] - 640364487 | 0; a = (a << 4 | a >>> 28) + b | 0; d += (a ^ b ^ c) + k[12] - 421815835 | 0; d = (d << 11 | d >>> 21) + a | 0; c += (d ^ a ^ b) + k[15] + 530742520 | 0; c = (c << 16 | c >>> 16) + d | 0; b += (c ^ d ^ a) + k[2] - 995338651 | 0; b = (b << 23 | b >>> 9) + c | 0; a += (c ^ (b | ~d)) + k[0] - 198630844 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[7] + 1126891415 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[14] - 1416354905 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[5] - 57434055 | 0; b = (b << 21 | b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[12] + 1700485571 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[3] - 1894986606 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[10] - 1051523 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[1] - 2054922799 | 0; b = (b << 21 | b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[8] + 1873313359 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[15] - 30611744 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[6] - 1560198380 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[13] + 1309151649 | 0; b = (b << 21 | b >>> 11) + c | 0; a += (c ^ (b | ~d)) + k[4] - 145523070 | 0; a = (a << 6 | a >>> 26) + b | 0; d += (b ^ (a | ~c)) + k[11] - 1120210379 | 0; d = (d << 10 | d >>> 22) + a | 0; c += (a ^ (d | ~b)) + k[2] + 718787259 | 0; c = (c << 15 | c >>> 17) + d | 0; b += (d ^ (c | ~a)) + k[9] - 343485551 | 0; b = (b << 21 | b >>> 11) + c | 0; x[0] = a + x[0] | 0; x[1] = b + x[1] | 0; x[2] = c + x[2] | 0; x[3] = d + x[3] | 0 } function md5blk(s) { var md5blks = [], i; for (i = 0; i < 64; i += 4) { md5blks[i >> 2] = s.charCodeAt(i) + (s.charCodeAt(i + 1) << 8) + (s.charCodeAt(i + 2) << 16) + (s.charCodeAt(i + 3) << 24) } return md5blks } function md5blk_array(a) { var md5blks = [], i; for (i = 0; i < 64; i += 4) { md5blks[i >> 2] = a[i] + (a[i + 1] << 8) + (a[i + 2] << 16) + (a[i + 3] << 24) } return md5blks } function md51(s) { var n = s.length, state = [1732584193, -271733879, -1732584194, 271733878], i, length, tail, tmp, lo, hi; for (i = 64; i <= n; i += 64) { md5cycle(state, md5blk(s.substring(i - 64, i))) } s = s.substring(i - 64); length = s.length; tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; for (i = 0; i < length; i += 1) { tail[i >> 2] |= s.charCodeAt(i) << (i % 4 << 3) } tail[i >> 2] |= 128 << (i % 4 << 3); if (i > 55) { md5cycle(state, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0 } } tmp = n * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(state, tail); return state } function md51_array(a) { var n = a.length, state = [1732584193, -271733879, -1732584194, 271733878], i, length, tail, tmp, lo, hi; for (i = 64; i <= n; i += 64) { md5cycle(state, md5blk_array(a.subarray(i - 64, i))) } a = i - 64 < n ? a.subarray(i - 64) : new Uint8Array(0); length = a.length; tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; for (i = 0; i < length; i += 1) { tail[i >> 2] |= a[i] << (i % 4 << 3) } tail[i >> 2] |= 128 << (i % 4 << 3); if (i > 55) { md5cycle(state, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0 } } tmp = n * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(state, tail); return state } function rhex(n) { var s = "", j; for (j = 0; j < 4; j += 1) { s += hex_chr[n >> j * 8 + 4 & 15] + hex_chr[n >> j * 8 & 15] } return s } function hex(x) { var i; for (i = 0; i < x.length; i += 1) { x[i] = rhex(x[i]) } return x.join("") } if (hex(md51("hello")) !== "5d41402abc4b2a76b9719d911017c592") { add32 = function (x, y) { var lsw = (x & 65535) + (y & 65535), msw = (x >> 16) + (y >> 16) + (lsw >> 16); return msw << 16 | lsw & 65535 } } if (typeof ArrayBuffer !== "undefined" && !ArrayBuffer.prototype.slice) { (function () { function clamp(val, length) { val = val | 0 || 0; if (val < 0) { return Math.max(val + length, 0) } return Math.min(val, length) } ArrayBuffer.prototype.slice = function (from, to) { var length = this.byteLength, begin = clamp(from, length), end = length, num, target, targetArray, sourceArray; if (to !== undefined) { end = clamp(to, length) } if (begin > end) { return new ArrayBuffer(0) } num = end - begin; target = new ArrayBuffer(num); targetArray = new Uint8Array(target); sourceArray = new Uint8Array(this, begin, num); targetArray.set(sourceArray); return target } })() } function toUtf8(str) { if (/[\u0080-\uFFFF]/.test(str)) { str = unescape(encodeURIComponent(str)) } return str } function utf8Str2ArrayBuffer(str, returnUInt8Array) { var length = str.length, buff = new ArrayBuffer(length), arr = new Uint8Array(buff), i; for (i = 0; i < length; i += 1) { arr[i] = str.charCodeAt(i) } return returnUInt8Array ? arr : buff } function arrayBuffer2Utf8Str(buff) { return String.fromCharCode.apply(null, new Uint8Array(buff)) } function concatenateArrayBuffers(first, second, returnUInt8Array) { var result = new Uint8Array(first.byteLength + second.byteLength); result.set(new Uint8Array(first)); result.set(new Uint8Array(second), first.byteLength); return returnUInt8Array ? result : result.buffer } function hexToBinaryString(hex) { var bytes = [], length = hex.length, x; for (x = 0; x < length - 1; x += 2) { bytes.push(parseInt(hex.substr(x, 2), 16)) } return String.fromCharCode.apply(String, bytes) } function SparkMD5() { this.reset() } SparkMD5.prototype.append = function (str) { this.appendBinary(toUtf8(str)); return this }; SparkMD5.prototype.appendBinary = function (contents) { this._buff += contents; this._length += contents.length; var length = this._buff.length, i; for (i = 64; i <= length; i += 64) { md5cycle(this._hash, md5blk(this._buff.substring(i - 64, i))) } this._buff = this._buff.substring(i - 64); return this }; SparkMD5.prototype.end = function (raw) { var buff = this._buff, length = buff.length, i, tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ret; for (i = 0; i < length; i += 1) { tail[i >> 2] |= buff.charCodeAt(i) << (i % 4 << 3) } this._finish(tail, length); ret = hex(this._hash); if (raw) { ret = hexToBinaryString(ret) } this.reset(); return ret }; SparkMD5.prototype.reset = function () { this._buff = ""; this._length = 0; this._hash = [1732584193, -271733879, -1732584194, 271733878]; return this }; SparkMD5.prototype.getState = function () { return { buff: this._buff, length: this._length, hash: this._hash } }; SparkMD5.prototype.setState = function (state) { this._buff = state.buff; this._length = state.length; this._hash = state.hash; return this }; SparkMD5.prototype.destroy = function () { delete this._hash; delete this._buff; delete this._length }; SparkMD5.prototype._finish = function (tail, length) { var i = length, tmp, lo, hi; tail[i >> 2] |= 128 << (i % 4 << 3); if (i > 55) { md5cycle(this._hash, tail); for (i = 0; i < 16; i += 1) { tail[i] = 0 } } tmp = this._length * 8; tmp = tmp.toString(16).match(/(.*?)(.{0,8})$/); lo = parseInt(tmp[2], 16); hi = parseInt(tmp[1], 16) || 0; tail[14] = lo; tail[15] = hi; md5cycle(this._hash, tail) }; SparkMD5.hash = function (str, raw) { return SparkMD5.hashBinary(toUtf8(str), raw) }; SparkMD5.hashBinary = function (content, raw) { var hash = md51(content), ret = hex(hash); return raw ? hexToBinaryString(ret) : ret }; SparkMD5.ArrayBuffer = function () { this.reset() }; SparkMD5.ArrayBuffer.prototype.append = function (arr) { var buff = concatenateArrayBuffers(this._buff.buffer, arr, true), length = buff.length, i; this._length += arr.byteLength; for (i = 64; i <= length; i += 64) { md5cycle(this._hash, md5blk_array(buff.subarray(i - 64, i))) } this._buff = i - 64 < length ? new Uint8Array(buff.buffer.slice(i - 64)) : new Uint8Array(0); return this }; SparkMD5.ArrayBuffer.prototype.end = function (raw) { var buff = this._buff, length = buff.length, tail = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], i, ret; for (i = 0; i < length; i += 1) { tail[i >> 2] |= buff[i] << (i % 4 << 3) } this._finish(tail, length); ret = hex(this._hash); if (raw) { ret = hexToBinaryString(ret) } this.reset(); return ret }; SparkMD5.ArrayBuffer.prototype.reset = function () { this._buff = new Uint8Array(0); this._length = 0; this._hash = [1732584193, -271733879, -1732584194, 271733878]; return this }; SparkMD5.ArrayBuffer.prototype.getState = function () { var state = SparkMD5.prototype.getState.call(this); state.buff = arrayBuffer2Utf8Str(state.buff); return state }; SparkMD5.ArrayBuffer.prototype.setState = function (state) { state.buff = utf8Str2ArrayBuffer(state.buff, true); return SparkMD5.prototype.setState.call(this, state) }; SparkMD5.ArrayBuffer.prototype.destroy = SparkMD5.prototype.destroy; SparkMD5.ArrayBuffer.prototype._finish = SparkMD5.prototype._finish; SparkMD5.ArrayBuffer.hash = function (arr, raw) { var hash = md51_array(new Uint8Array(arr)), ret = hex(hash); return raw ? hexToBinaryString(ret) : ret }; return SparkMD5 });
前端html
<div style="margin: 150px auto;text-align: center">
<div>
<h2 >上传text批量导入kafka</h2>
</div>
<div style="padding-left: 678px;">
<div style="float: left;">
<!-- 进度条html元素 -->
<progress id="myProgress" value="0" max="100" style="width: 350px;height: 25px"></progress> <span id="mySpan">0%</span>
</div>
<div style="float: left;">
<!-- 上传按钮 -->
<button id="kafkaTextFile">上传文件</button>
</div>
</div>
</div>
前端js
// 进度条
function Progress(value) {
var myProgress = document.getElementById("myProgress");
var mySpan = document.getElementById("mySpan");
mySpan.innerText = value + "%";
myProgress.value = value;
}
$.fcup({
upId : 'kafkaTextFile', // 上传dom的id
upShardSize : '10', // 切片大小,(单次上传最大值)单位M,默认2M
upMaxSize : '1000', // 上传文件大小,单位M,不设置不限制
upUrl : '/v1/dataManager/uploadKafkaTask', // 文件上传接口
upType : 'txt', // 上传类型检测,用,号分割
// 接口返回结果回调,根据结果返回的数据来进行判断,可以返回字符串或者json来进行判断处理
upCallBack : function(res) {
// 状态
var status = res.data;
var code = res.code;
// 信息
var msg = res.message;
// url
// 已经完成了
if (status == 2) {
alert(msg);
location.reload();
}
// 还在上传中
if (status == 1) {
console.log(msg);
}
// 接口返回错误
if (code != 200) {
// 停止上传并且提示信息
$.upStop(msg + "," + status);
location.reload();
}
},
// 上传过程监听,可以根据当前执行的进度值来改变进度条
upEvent : function(num) {
// num的值是上传的进度,从1到100
Progress(num);
},
// 发生错误后的处理
upStop : function(errmsg) {
// 这里只是简单的alert一下结果,可以使用其它的弹窗提醒插件
alert(errmsg);
},
// 开始上传前的处理和回调,比如进度条初始化等
upStart : function() {
Progress(0);
$('#pic').hide();
alert('开始上传');
}
});