使用场景
无法直接登录服务器上传文件，使用web端上传超大文件出现超时
实现原理
上传
server端与client端建立websocket连接，client将待传文件进行分块，然后将文件的相关信息(文件名、md5值、分块大小、总块数、当前块数)及文件数据一并上传到服务端，服务端在本地建立文件通过追加的方式将上传的数据写入文件中，当当前块与总块数相等且文件MD5相同时认为文件上传成功
下载
与上传相反，将client当成服务端，client与server建立连接后，向服务端发送可接收请求，服务端收到后将文件进行分块处理并记录文件相关信息连同数据一并发送给client端，当服务端发现文件读取完毕后将块大小设置为-1，client端读取后进行文件md5校验，校验通过则认为下载成功。
代码展示
jdk版本: 14
项目地址: https://gitee.com/LovingL/big-file-upload-project
使用依赖
服务端
dependencies {
implementation('org.springframework.boot:spring-boot-starter-undertow')
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation('org.springframework.boot:spring-boot-starter-websocket') {
exclude module: 'spring-boot-starter-tomcat'
}
implementation 'cn.hutool:hutool-json:5.6.5'
implementation 'cn.hutool:hutool-crypto:5.6.5'
compileOnly 'org.projectlombok:lombok'
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.taobao.arthas:arthas-spring-boot-starter:3.4.8'
annotationProcessor 'org.projectlombok:lombok'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
}
}
客户端
dependencies {
implementation("org.java-websocket:Java-WebSocket:1.5.1")
implementation("cn.hutool:hutool-json:5.4.3")
implementation("cn.hutool:hutool-crypto:5.4.3")
implementation("org.slf4j:slf4j-jdk14:1.7.25")
compileOnly("org.projectlombok:lombok:1.18.12")
annotationProcessor("org.projectlombok:lombok:1.18.12")
testCompile group: 'junit', name: 'junit', version: '4.12'
}
消息定义
文件发送消息:
public class SendFileData {
private String fileName; //文件名
private int currentBlock; //当前块号
private int totalBlock; //总块号
private int blockSize; //块大小
private String md5sum; //md5值
private byte[] data; //文件数据
}
文件接收消息:
public class ReceiveFileData {
private int status; //接收状态码
private String errMsg; //错误消息返回
private String uri; //文件地址
}
文件上传
服务端
public void onMessage(Session session, String reqData) throws IOException {
SendFileData data = JSONUtil.toBean(reqData, SendFileData.class);
String filePath = StrUtil.format("{}/{}", fileConfig.getTempDir(), data.getFileName());
if (file == null) {
file = FileUtil.file(filePath);
if (!FileUtil.exist(file)) {
FileUtil.touch(file);
}
md5sum = MD5.create().digestHex16(file);
}
ReceiveFileData receiveFileData = new ReceiveFileData();
//先进行md5校验，如果md5已相同说明已经上传过了就直接返回服务端地址即可
if (md5sum.equals(data.getMd5sum())) {
receiveFileData.setStatus(HttpStatus.OK.value());
receiveFileData.setUri(filePath);
session.getBasicRemote().sendText(JSONUtil.toJsonStr(receiveFileData));
return;
}
int currentBlock = data.getCurrentBlock();
if (totalBlocks == null) {
totalBlocks = data.getTotalBlock();
}
int blockSize = data.getBlockSize();
//通过追加方式将数据写入文件
FileUtil.writeBytes(data.getData(), file, 0, blockSize, true);
//当前块号与总块号相等时判断MD5是否相同
if (currentBlock == totalBlocks) {
md5sum = MD5.create().digestHex16(file);
if (md5sum.equals(data.getMd5sum())) {
receiveFileData.setStatus(HttpStatus.OK.value());
receiveFileData.setUri(filePath);
} else {
receiveFileData.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
receiveFileData.setErrMsg("MD5计算异常，文件上传异常");
}
} else {
receiveFileData.setStatus(HttpStatus.ACCEPTED.value());
}
log.info("cur:{}, tot:{}", currentBlock, totalBlocks);
session.getBasicRemote().sendText(JSONUtil.toJsonStr(receiveFileData));
}
客户端
try (InputStream inputStream = FileUtil.getInputStream(file)) {
URI uri = new URI(this.sendUrl);
MyWebSocketClient webSocketClient = new MyWebSocketClient(uri, this.logInfo);
webSocketClient.connectBlocking();
SendFileData sendFileData = new SendFileData();
long fileLength = file.length();
int currentBlock = 1;
//计算文件总块数
int totalBlocks = (int) Math.ceil((double) fileLength / Double.valueOf(4096));
sendFileData.setFileName(file.getName());
sendFileData.setTotalBlock(totalBlocks);
sendFileData.setMd5sum(MD5.create().digestHex16(file));
this.logInfo.append("总块数:" + totalBlocks + "\n");
byte[] b = new byte[4096];
int i = 0;
String message = null;
StringBuffer stringBuffer = new StringBuffer();
while ((i = inputStream.read(b)) != -1) {
byte[] data = new byte[i];
sendFileData.setBlockSize(i);
System.arraycopy(b, 0, data, 0, i); //最后读取部分不为固定大小，按照实际大小拷贝不然md5将不同
sendFileData.setData(data);
sendFileData.setCurrentBlock(currentBlock);
message = webSocketClient.sendAndGet(JSONUtil.toJsonStr(sendFileData));
if (message != null) {
ReceiveFileData receiveFileData = JSONUtil.toBean(message, ReceiveFileData.class);
//当服务器返回值为200说明已经接收完成
if (receiveFileData.getStatus() == 200) {
this.logInfo.append("文件所在服务器路径为:" + receiveFileData.getUri() + "\n");
webSocketClient.close();
break;
}
if (receiveFileData.getStatus() == 500) {
this.logInfo.append("文件上传异常:" + receiveFileData.getErrMsg() + "\n");
webSocketClient.close();
break;
}
}
currentBlock++;
}
} catch (IOException ioException) {
this.logInfo.append("异常:文件打开异常\n");
} catch (URISyntaxException uriSyntaxException) {
this.logInfo.append("异常:不可识别的URI地址\n");
} catch (InterruptedException interruptedException) {
this.logInfo.append("异常:连接被中断\n");
} catch (JSONException jsonException) {
this.logInfo.append("异常:服务器返回值无法解析:" + jsonException.getMessage() + "\n");
}
文件下载
服务端
public void onMessage(Session session, String receiveStringData) throws IOException {
ReceiveFileData receiveFileData = JSONUtil.toBean(receiveStringData, ReceiveFileData.class);
if (receiveFileData.getStatus() == HttpStatus.ACCEPTED.value()) {
if (file == null) {
file = new File(receiveFileData.getUri());
}
if (fileLength == null) {
fileLength = file.length();
}
SendFileData sendFileData = new SendFileData();
if (totalBlocks == null) {
totalBlocks = (int) Math.ceil((double) fileLength / Double.valueOf(blockSize));
}
sendFileData.setFileName(file.getName());
sendFileData.setTotalBlock(totalBlocks);
if (md5Sum == null) {
md5Sum = MD5.create().digestHex16(file);
}
sendFileData.setMd5sum(md5Sum);
//使用RandomAccessFile来将文件指针定位到当前块号所处位置
try(RandomAccessFile randomAccessFile = new RandomAccessFile(file.getAbsolutePath(), "r");) {
byte[] b = new byte[blockSize];
randomAccessFile.seek((currentBlock - 1) * blockSize);
int realBlockSize = randomAccessFile.read(b, 0, blockSize);
log.info("totalBlocks:{} currentBlock:{}, realBlockSize:{}", totalBlocks, currentBlock, realBlockSize);
if (currentBlock > totalBlocks || realBlockSize == -1) {
log.info("文件已读取完毕");
} else {
byte[] data = new byte[realBlockSize];
sendFileData.setBlockSize(realBlockSize);
System.arraycopy(b, 0, data, 0, realBlockSize); //一定要按照实际大小进行拷贝不然MD5值将不同
sendFileData.setData(data);
sendFileData.setCurrentBlock(currentBlock);
currentBlock++;
}
sendFileData.setBlockSize(realBlockSize);
session.getBasicRemote().sendText(JSONUtil.toJsonStr(sendFileData));
} catch (IOException ioException) {
ioException.printStackTrace();
}
} else {
session.getBasicRemote().sendText("发送完成");
session.close(new CloseReason(CloseReason.CloseCodes.NORMAL_CLOSURE, "发送完成"));
}
}
客户端
try {
URI uri = new URI(this.downloadUrl);
String name = FileUtil.getName(fileUriText);
MyWebSocketClient webSocketClient = new MyWebSocketClient(uri, logInfo2);
webSocketClient.connectBlocking();
ReceiveFileData receiveFileData = new ReceiveFileData();
//开始设置202状态告知服务器客户端已准备接收
receiveFileData.setStatus(202);
receiveFileData.setUri(fileUriText);
File downloadFile = FileUtil.file(saveDir, name);
if (!FileUtil.exist(downloadFile)) {
FileUtil.touch(downloadFile);
}
while (true) {
String md5Sum = MD5.create().digestHex16(downloadFile);
String response = webSocketClient.sendAndGet(JSONUtil.toJsonStr(receiveFileData));
if (response != null) {
SendFileData sendFileData = JSONUtil.toBean(response, SendFileData.class);
if (md5Sum.equals(sendFileData.getMd5sum())) {
this.logInfo2.append("接收完成\n");
webSocketClient.close(1000, "接收完成");
break;
}
if (sendFileData.getBlockSize() == -1) {
this.logInfo2.append("异常:与远程MD5计算不一致\n");
webSocketClient.close(1006, "MD5异常");
break;
} else {
//通过追加方式将文件数据写入文件当中
FileUtil.writeBytes(sendFileData.getData(), downloadFile, 0, sendFileData.getBlockSize(), true);
}
}
}
} catch (URISyntaxException uriSyntaxException) {
this.logInfo2.append("异常:远程服务器地址异常");
} catch (InterruptedException interruptedException) {
this.logInfo2.append("异常:程序异常中断");
}
677
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
