aliyun多线程分片上传

POM.xml

文档地址

<dependency>
	<groupId>com.aliyun.oss</groupId>
	<artifactId>aliyun-sdk-oss</artifactId>
	<version>2.8.3</version>
</dependency>

逻辑实现

1. 配置文件

@ConfigurationProperties(prefix = "system")
@Setter
@Getter
@Component
@RefreshScope
public class SysProperties {
    private String endpoint;
    private String endpointInternal;
    private String accessKeyId;
    private String accessKeySecret;
    private String bucketName;
    private String callbackUrl;
    private String dir;
    private String mpsRegionId; // 地域ID
    private String m3u8TemplateId;
    private String m3u8PipelineId;

}

@Component
public class OSSClientInit {

    @Autowired
    private SysProperties sysProperties;

    public OSSClient getOSSClient() {

        OSSClient ossClient = new OSSClient(sysProperties.getEndpointInternal(), sysProperties.getAccessKeyId(), sysProperties.getAccessKeySecret());
        return ossClient;
    }
}

2. 逻辑代码

@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PartEtagDto implements Serializable {

    private static final long serialVersionUID = 3346126482134611901L;

    private int partNumber;

    private String partETag;

}

@Data
@NoArgsConstructor
@Builder
@AllArgsConstructor
public class MultiPartUploadThread implements Callable<MultiPartUploadThread>, Serializable {
    private static final long serialVersionUID = 6503131120165475652L;

    private MultipartFile uploadFile;
    private String bucket;
    private String fileName;
    private long start;
    private long size;
    private int partId;
    private String uploadId;
    private OSSClient client;

    private PartEtagDto partEtagDto;

    public MultiPartUploadThread(OSSClient client, String bucket, String fileName,
                            MultipartFile uploadFile, String uploadId, int partId,
                            long start, long partSize) {
        this.client = client;
        this.uploadFile = uploadFile;
        this.bucket = bucket;
        this.fileName = fileName;
        this.start = start;
        this.size = partSize;
        this.partId = partId;
        this.uploadId = uploadId;
    }



    @Override
    public MultiPartUploadThread call() {
        InputStream in = null;
        try {
            in = uploadFile.getInputStream();
            in.skip(start);

            UploadPartRequest uploadPartRequest = new UploadPartRequest();
            uploadPartRequest.setBucketName(bucket);
            uploadPartRequest.setKey(fileName);
            uploadPartRequest.setUploadId(uploadId);
            uploadPartRequest.setInputStream(in);
            uploadPartRequest.setPartSize(size);
            uploadPartRequest.setPartNumber(partId);
            // PartEtagDto是对uploadPartResult.getPartETag()的返回值PartETag的封装，主要是为了能序列化PartETag，PartEtagDto仅比PartETag多实现了Serializable接口
            UploadPartResult uploadPartResult = client.uploadPart(uploadPartRequest);
            partEtagDto = new PartEtagDto(uploadPartResult.getPartETag().getPartNumber(), uploadPartResult.getPartETag().getETag());
        } catch (Exception e) {
            e.printStackTrace();

        } finally {
            IOUtils.closeQuietly(in);
        }
        return this;
    }
}

@Data
@AllArgsConstructor
@NoArgsConstructor
public class MultiUploadPartObj implements Serializable {
    private static final long serialVersionUID = -4943456905327235650L;
    private List<MultiPartUploadThread> uploadPartThreads = Collections.synchronizedList(new ArrayList<MultiPartUploadThread>());

    private boolean result = true;

}

public class MultipartUploadServiceImpl {

    @Autowired
    private OSSClientInit ossClientInit;

    @Autowired
    private SysProperties sysProperties;


    @PostMapping("/uploadFile/multipartUpload")
    public RestResponse multipartUploadFile(@RequestBody UploadFileDto uploadFileDto) {
        OSSClient ossClient = ossClientInit.getOSSClient();

        String fileName = uploadFileDto.getFileName();
        MultipartFile multipartFile = uploadFileDto.getFile();

        Integer partSize = 2 * 1024 * 1024; // 2M
        int partCount = calPartCount(multipartFile, partSize); // 计算文件有多少个分片。

        String uploadId;

        MultiUploadPartObj uploadPartObj = new MultiUploadPartObj();
        try {
            //初始化MultipartUpload 返回uploadId
            uploadId = initMultipartUpload(ossClient, sysProperties.getBucketName(), fileName);
        } catch (Exception e) {
            e.printStackTrace();

            return RestResponse.error(RestCode.UNKNOWN_ERROR);
        }
        for (int i = 0; i < partCount ; i++) {
            long start = partSize * i;
            long curPartSize = (i + 1 == partCount) ? (multipartFile.getSize() - start) : partSize;
            //构造上传线程，UploadPartThread是执行每个分块上传任务的线程
            uploadPartObj.getUploadPartThreads().add(new MultiPartUploadThread(ossClient, sysProperties.getBucketName(), fileName, multipartFile, uploadId, i + 1, start, curPartSize));
        }

        try {
            int i = 0;
            // upload方法提交分块上传线程至子线程池上传，while循环用于上传失败重复上传，Constant.RETRY定义重复次数
            while (!upload(uploadPartObj).isResult()) {
                if(++i == 3) {
                    break;
                }
            }
        } catch (Exception e) {
            log.info("上传失败");
            return RestResponse.error(RestCode.UNKNOWN_ERROR);
        }

        if(!uploadPartObj.isResult()){
            return RestResponse.error(RestCode.UNKNOWN_ERROR);
        }

        try {
            //完成一个multi-part请求。
            completeMultipartUpload(ossClient, sysProperties.getBucketName(), fileName, uploadPartObj);
        } catch (Exception e) {
            log.error("multi-part请求失败！！");
            return RestResponse.error(RestCode.UNKNOWN_ERROR);
        }
        return RestResponse.success();
    }

    private String initMultipartUpload(OSSClient client,String bucketName, String fileName) throws Exception{
        String uploadId=null;
        try{
            InitiateMultipartUploadRequest initUploadRequest = new InitiateMultipartUploadRequest(bucketName, fileName);
            InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initUploadRequest);
            uploadId = initResult.getUploadId();

            System.err.println(uploadId);
        }catch (Exception e) {
            log.info("初始化Multi-part upload请求失败！！");
        }
        return uploadId;

    }


    // 根据文件的大小和每个Part的大小计算需要划分的Part个数。
    private int calPartCount(MultipartFile multipartFile, Integer partSize) {
        int partCount = (int) (multipartFile.getSize() / partSize);
        if (multipartFile.getSize() % partSize != 0){
            partCount++;
        }
        return partCount;
    }

    private MultiUploadPartObj upload(MultiUploadPartObj uploadPartObj){
        ExecutorService pool = getExecutorService();
        try {

            uploadPartObj.setResult(true);
            //向子线程池中submit单个文件所有分块上传线程
            for (int i=0;i<uploadPartObj.getUploadPartThreads().size();i++) {
                if(uploadPartObj.getUploadPartThreads().get(i).getPartEtagDto()==null) {
                    pool.submit(uploadPartObj.getUploadPartThreads().get(i));
                }
            }

            //shutdown子线程池，池内所上传任务执行结束后停止当前线程池
            pool.shutdown();
            while (!pool.isTerminated()) {
                pool.awaitTermination(10, TimeUnit.SECONDS);
            }

            //判断上传结果
            for (MultiPartUploadThread uploadPartThread: uploadPartObj.getUploadPartThreads()) {
                if(uploadPartThread.getPartEtagDto()==null) {
                    uploadPartObj.setResult(false);
                }

            }
        } catch (Exception e) {
            log.error("多线程上传单个文件异常！！");

        }
        return uploadPartObj;
    }

    // 完成一个multi-part请求。
    private void completeMultipartUpload(OSSClient client, String bucketName, String key,MultiUploadPartObj uploadPartObj){
        List<PartETag> eTags = new ArrayList<>();
        for (MultiPartUploadThread uploadPartThread : uploadPartObj.getUploadPartThreads()) {
            eTags.add(new PartETag(uploadPartThread.getPartEtagDto().getPartNumber(),uploadPartThread.getPartEtagDto().getPartETag()));
        }
        // 为part按 partNumber 排序
        eTags.sort(Comparator.comparingInt(PartETag::getPartNumber));
        try{
            CompleteMultipartUploadRequest completeMultipartUploadRequest =
                    new CompleteMultipartUploadRequest(bucketName, key, uploadPartObj.getUploadPartThreads().get(0).getUploadId(), eTags);
            client.completeMultipartUpload(completeMultipartUploadRequest);
        }catch (Exception e) {
            log.error("合并上传失败！！");
        }
    }

    /**
     * 获取自定义线程池
     * @return ExecutorService
     */
    private static ExecutorService getExecutorService(){

        ThreadFactory threadName = new ThreadFactoryBuilder().setNameFormat("pool-%d").build();

        return new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 5,
                100,
                10L,
                TimeUnit.SECONDS,
                new LinkedBlockingQueue<Runnable>(1000),
                threadName,
                new ThreadPoolExecutor.AbortPolicy());
    }
}