阿里云OSS能获取进度的简单上传及线程池实现分片上传
application.yml
#阿里OSS分片上传配置
aliOss:
multipartUpload:
partCount: 100 # 分片数量
threshold: 524288000 # 启用分片上传阈值：500M 500 * 1024 * 1024
threadCount: 3 # 线程数
controller层
@RestController
@RequestMapping("/api/import")
public class ImportController {
@Resource
private ImportService importService;
@Resource
private OssService ossService;
private static final Logger logger = LoggerFactory.getLogger(ImportController.class);
@PostMapping("/upload")
@ResponseBody
public CommonResult<String> upload(MultipartFile file, HttpServletRequest request, HttpSession session) throws IOException {
return importService.upload(file, session);
}
/**
* 简单列举分片上传事件
* @param request
* @return
*/
@RequestMapping ("/listMultipartUploads")
@ResponseBody
public CommonResult<String> listMultipartUploads(HttpServletRequest request, HttpSession session) {
return importService.listMultipartUploads(session);
}
/**
* 通过fileName获取实时长传进度
* @param request
* @return
*/
@RequestMapping ("/percent")
@ResponseBody
public CommonResult<String> getUploadPercent(HttpServletRequest request, String fileName) {
HttpSession session = request.getSession();
String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
Double percent = session.getAttribute(name) == null ? 0 : Double.valueOf(session.getAttribute(name).toString());
logger.debug(name + ":" + session.getAttribute(name));
return new CommonResult<>(String.valueOf(percent));
}
/**
* 重置上传进度
* @param request
* @return
*/
@PostMapping("/percent/reset")
public CommonResult resetPercent(HttpServletRequest request, String fileName) {
HttpSession session = request.getSession();
String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
session.setAttribute(name, 0);
return new CommonResult(CommonResultEmnu.OK);
}
/**
* @Description: OSS删除
*/
@RequestMapping ("/ossDel")
public CommonResult ossDel(String filePath) {
ossService.OssDel(filePath);
return new CommonResult(CommonResultEmnu.OK);
}
}
service层
@Service
@Primary
@Slf4j
public class ImportServiceOss implements ImportService {
@Value("${oss.filePrefix}")
private String filePrefix;
/**
* 阈值，达到阈值使用分片上传
*/
@Value("${aliOss.multipartUpload.threshold}")
private int threshold;
@Override
public CommonResult<String> upload(MultipartFile file, HttpSession session) {
if (file == null) {
return new CommonResult<>(CommonResultEmnu.ERROR, "文件为空");
}
if (!file.isEmpty()) {
String fileName = file.getOriginalFilename();
if (fileName == null) {
return new CommonResult<>(CommonResultEmnu.INVALID_PARAMS, "获取文件名失败");
}
// InputStream inputStream = file.getInputStream();
String suffix = fileName.substring(fileName.lastIndexOf(".") + 1);
String newName = IdGenerator.getUUID() + "." + suffix.toLowerCase(Locale.ENGLISH);
String filePath = filePrefix + "_upload" + ResourceUtil.createFilePath().replaceAll("/", "_") + newName;
try {
//MultipartFile转File
File newFile = new File(fileName);
FileOutputStream os = new FileOutputStream(newFile);
os.write(file.getBytes());
os.close();
//将上传文件写入目标文件
file.transferTo(newFile);
//大于阈值走分片上传，否则直接简单上传
long length = newFile.length();
if (length < threshold) {
return this.simpleUpload(newFile, session, fileName, filePath);
} else {
return this.multipartUpload(newFile, session, fileName, filePath);
}
} catch (Exception e) {
log.error("MultipartFile转File失败： ", e);
return new CommonResult<>(CommonResultEmnu.ERROR, "MultipartFile转File失败！");
}
} else {
return new CommonResult<>(CommonResultEmnu.ERROR, "上传失败，因为文件是空的");
}
}
/**
* 简单上传
*/
public CommonResult<String> simpleUpload(File f, HttpSession session, String oldFileName, String filePath) {
// 创建OSSClient实例。
OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET);
// 异步处理，可以使用线程池
Thread t = new Thread(() -> {
try {
// 带进度条的上传。
ossClient.putObject(new PutObjectRequest(OssUtil.getBucketName(filePath), filePath, f).
withProgressListener(new PutObjectProgressListener(session, oldFileName)));
} catch (Exception e) {
log.error("上传OSS文件出现异常：", e);
} finally {
//保证删除服务器上临时文件
f.delete();
// 关闭OSSClient。
ossClient.shutdown();
}
}, "简单上传-" + oldFileName);
t.start();
String url = OssUtil.getUrl(filePath);
log.info("upload_url{} ", url);
return new CommonResult<>(url);
}
/**
* 分片上传
*/
public CommonResult<String> multipartUpload(File newFile, HttpSession session, String oldFileName, String filePath) {
// 异步处理，可以使用线程池
Thread t = new Thread(() -> {
try {
AliyunOSSUtil.upLoad(newFile, session, filePath, oldFileName);
} catch (Throwable e) {
log.error("上传OSS文件出现异常：", e);
} finally {
//保证删除服务器上临时文件
newFile.delete();
}
});
t.start();
String url = OssUtil.getUrl(filePath);
log.info("upload_url:{} ", url);
return new CommonResult<>(url);
}
/**
* 查询所有未完成的分片上传信息
*/
@Override
public CommonResult<String> listMultipartUploads(HttpSession session) {
// 创建OSSClient实例。
OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET);
String bucketName = OssUtil.getBucketName(filePrefix + "_");
// 列举分片上传事件。默认列举1000个分片。
ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
MultipartUploadListing multipartUploadListing = ossClient.listMultipartUploads(listMultipartUploadsRequest);
for (MultipartUpload multipartUpload : multipartUploadListing.getMultipartUploads()) {
// 获取uploadId。
multipartUpload.getUploadId();
// 获取Key。
multipartUpload.getKey();
// 获取分片上传的初始化时间。
multipartUpload.getInitiated();
}
// 关闭OSSClient。
ossClient.shutdown();
return null;
}
简单上传的自定义监听器，用于获取进度
@Slf4j
public class PutObjectProgressListener implements ProgressListener {
private long bytesWritten = 0;
private long totalBytes = -1;
private boolean succeed = false;
private HttpSession session;
private String fileName;
private static final Logger logger = LoggerFactory.getLogger(PutObjectProgressListener.class);
//构造方法中加入session
public PutObjectProgressListener() {
}
public PutObjectProgressListener(HttpSession mSession, String fileName) {
this.session = mSession;
int percent = 0;
this.fileName = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
session.setAttribute(this.fileName, percent);
}
@Override
public void progressChanged(ProgressEvent progressEvent) {
long bytes = progressEvent.getBytes();
ProgressEventType eventType = progressEvent.getEventType();
switch (eventType) {
case TRANSFER_STARTED_EVENT:
logger.debug("Start to upload......");
break;
case REQUEST_CONTENT_LENGTH_EVENT:
this.totalBytes = bytes;
logger.debug(this.totalBytes + " bytes in total will be uploaded to OSS");
break;
case REQUEST_BYTE_TRANSFER_EVENT:
this.bytesWritten += bytes;
if (this.totalBytes != -1) {
int percent = (int) (this.bytesWritten * 100.0 / this.totalBytes);
//将进度percent放入session中 官网demo中没有放入session这一步
session.setAttribute(this.fileName, percent);
logger.debug(bytes + " bytes have been written at this time, upload progress: " + percent + "%(" + this.bytesWritten + "/" + this.totalBytes + ")");
} else {
logger.debug(bytes + " bytes have been written at this time, upload ratio: unknown" + "(" + this.bytesWritten + "/...)");
}
break;
case TRANSFER_COMPLETED_EVENT:
this.succeed = true;
logger.debug("Succeed to upload, " + this.bytesWritten + " bytes have been transferred in total");
break;
case TRANSFER_FAILED_EVENT:
logger.debug("Failed to upload, " + this.bytesWritten + " bytes have been transferred");
break;
default:
break;
}
}
public boolean isSucceed() {
return succeed;
}
可以获取进度的分片上传方法实现
@Component
public class AliyunOSSUtil {
/**
* 分片数量，给静态变量用@Value赋值 需要写在set方法上
*/
private static int defaultPartCount;
/**
* 线程数量
*/
private static int threadCount;
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(AliyunOSSUtil.class);
/**
* 上传文件
**/
public static void upLoad(File file, HttpSession session, String filePath, String oldFileName) {
//创建一个可重用固定线程数的无界队列线程池 最多一百零一个任务，无需担心队列存放任务太多
ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
//判断文件
if (file == null) {
return;
}
ClientBuilderConfiguration conf = new ClientBuilderConfiguration();
//连接空闲超时时间，超时则关闭
conf.setIdleConnectionTime(2000);
//创建OSSClient实例，需要放到这里，不能把OSS作为静态变量，如果多个客户同时上传，客户A关闭ossClient，而客户B正用着ossClient
OSS ossClient = new OSSClientBuilder().build(OSSConfig.ENDPOINT, OSSConfig.ACCESSKEYID, OSSConfig.ACCESSKEYSECRET, conf);
//记录上传片数，线程安全，不能静态变量，客户A和客户B同时上传，片数到最后大于partSize
AtomicInteger uploadedPart = new AtomicInteger(0);
/*分片上传*/
/*1.初始化一个分片上传事件*/
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(OssUtil.getBucketName(filePath), filePath);
InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
String uploadId = result.getUploadId();
// session.setAttribute("uploadId", uploadId);
try {
/*2.上传分片**/
long partSize;
long fileLength = file.length();
//按默认片数计算每片大小
partSize = (int) fileLength / defaultPartCount;
//如果100片不能分完，就是101片
int partCount;
if (fileLength % partSize != 0) {
partCount = defaultPartCount + 1;
} else {
partCount = defaultPartCount;
}
//返回uploadId，它是分片上传事件的唯一标识，可以根据这个ID来发起相关操作，如取消分片上传、查询分片上传等
// 装饰者模式加锁，保证线程安全
List<PartETag> partETags = Collections.synchronizedList(new ArrayList<>(partCount));
//遍历分片上传
for (int i = 0; i < partCount; i++) {
long startPos = i * partSize;
//是否为最后一块分片
long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
//getBucketName(filePath)是Bucket的名字
executorService.execute(new PartUploader(ossClient, OssUtil.getBucketName(filePath), file, startPos, curPartSize, i + 1, uploadId, partETags, partCount, session, filePath, oldFileName,uploadedPart));
}
//等待所有的分片完成 执行后不再接收新任务，如果里面有任务，就执行完
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
logger.error(e.getMessage());
}
}
//重新设为0片
uploadedPart.set(0);
//验证是否所有的分片都完成
if (partETags.size() != partCount) {
throw new IllegalStateException("文件的某些部分上传失败！");
}
/*3.完成分片上传**/
//排序。partETags必须按分片号升序排列
partETags.sort(new Comparator<PartETag>() {
@Override
public int compare(PartETag o1, PartETag o2) {
return o1.getPartNumber() - o2.getPartNumber();
}
});
//在执行该操作时，需要提供所有有效的partETags。OSS收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，OSS将把这些分片组合成一个完整的文件。
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(OssUtil.getBucketName(filePath), filePath, uploadId, partETags);
ossClient.completeMultipartUpload(completeMultipartUploadRequest);
/*//设置权限
ossClient.setBucketAcl(OssUtil.getBucketName(filePath), CannedAccessControlList.PublicRead);*/
} catch (OSSException oe) {
logger.error(oe.getMessage());
} catch (ClientException ce) {
logger.error(ce.getErrorMessage());
} finally {
//关闭OSSClient
ossClient.shutdown();
}
}
/**
* 实现并启动线程
**/
private static class PartUploader implements Runnable {
private final String bucketName;
private final int partCount;
private final File localFile;
private final long startPos;
private final long partSize;
private final int partNumber;
private final String uploadId;
private List<PartETag> partETags;
private final HttpSession session;
private final String key;
private final String oldFileName;
private final OSS ossClient;
private AtomicInteger uploadedPart;
public PartUploader(OSS ossClient, String bucketName, File localFile, long startPos, long partSize, int partNumber, String uploadId, List<PartETag> partETags, int partCount, HttpSession session, String key, String oldFileName, AtomicInteger uploadedPart) {
this.ossClient = ossClient;
this.bucketName = bucketName;
this.localFile = localFile;
this.startPos = startPos;
this.partNumber = partNumber;
this.uploadId = uploadId;
this.partSize = partSize;
this.partETags = partETags;
this.partCount = partCount;
this.session = session;
this.key = key;
this.oldFileName = oldFileName;
this.uploadedPart = uploadedPart;
}
@Override
public void run() {
Thread.currentThread().setName("文件分片上传pool-" + oldFileName + "-" + partNumber);
InputStream inputStream = null;
try {
inputStream = new FileInputStream(this.localFile);
//跳过已经上传的分片
inputStream.skip(startPos);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucketName);
uploadPartRequest.setKey(key);
uploadPartRequest.setUploadId(this.uploadId);
uploadPartRequest.setInputStream(inputStream);
//设置分片大小。除了最后一个分片没有大小限制，其他分片最小为100KB
uploadPartRequest.setPartSize(this.partSize);
//设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出这个范围,OSS将返回InvalidArgum的错误码
uploadPartRequest.setPartNumber(this.partNumber);
//每个分片不需要按顺序上传，甚至可以在不同客户端上传，OSS会根据分片号排序组成完整的文件。
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
// System.out.println("Part#" + this.partNumber + "done");
// System.out.println();
//已经上传的片数加1
uploadedPart.incrementAndGet();
//每次上传分片之后，OSS的返回结果会包含一个PartETag。PartETag将被保存到PartETags中。
this.partETags.add(uploadPartResult.getPartETag());
//计算进度
int percent = (int) (uploadedPart.intValue() * 100.0 / partCount);
session.setAttribute(oldFileName, percent);
} catch (Exception e) {
logger.error(e.getMessage());
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
logger.error(e.getMessage());
}
}
}
}
}
@Value("${aliOss.multipartUpload.partCount}")
public void setDefaultPartCount(int defaultPartCount) {
AliyunOSSUtil.defaultPartCount = defaultPartCount;
}
@Value("${aliOss.multipartUpload.threadCount}")
public void setThreadCount(int threadCount) {
AliyunOSSUtil.threadCount = threadCount;
}
获取进度条接口，前端可以循环调用以下接口，传入文件名，获取进度
@RequestMapping ("/percent")
@ResponseBody
public CommonResult<Object> getUploadPercent(HttpServletRequest request, String fileName) {
HttpSession session = request.getSession();
String name = StringUtil.isNotBlank(fileName) ? fileName : "upload_percent";
return new CommonResult<>(session.getAttribute(name));
}
794
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
