Springboot集成华为云obs分段并发上传

1.pom引入华为云配置

    <dependency>
        <groupId>com.huaweicloud</groupId>
        <artifactId>esdk-obs-java-bundle</artifactId>
        <version>3.23.3</version>
    </dependency>

2.application.yml引入华为云配置

#华为云
hwyun:
  obs:
    #OBS存储桶名称
    bucketName:
    #华为云OBS服务节点，如上海（China East）
    endPoint:
    #华为云OBS账户AK
    accessKey:
    #华为云OBS账户SK
    secretKey:

3.添加配置文件HweiOBSConfig

import com.obs.services.ObsClient;
import com.obs.services.exception.ObsException;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;

import java.text.SimpleDateFormat;
import java.util.Date;
/**
 * @ClassName: HweiOBSConfig
 * @Description: 华为云OBS配置类
 */
@Data
@Slf4j
@Configuration
public class HweiOBSConfig {
    /**
     * 访问密钥AK
     */
    @Value("${hwyun.obs.accessKey}")
    private String accessKey;

    /**
     * 访问密钥SK
     */
    @Value("${hwyun.obs.secretKey}")
    private String secretKey;

    /**
     * 终端节点
     */
    @Value("${hwyun.obs.endPoint}")
    private String endPoint;

    /**
     * 桶
     */
    @Value("${hwyun.obs.bucketName}")
    private String bucketName;

    /**
     * @return
     * @Description 获取OBS客户端实例
     * @return: com.obs.services.ObsClient
     */
    public ObsClient getInstance() {
        return new ObsClient(accessKey, secretKey, endPoint);
    }


    /**
     * @return
     * @Description 销毁OBS客户端实例
     * @param: obsClient
     */
    public void destroy(ObsClient obsClient) {
        try {
            obsClient.close();
        } catch (ObsException e) {
            log.error("obs执行失败", e);
        } catch (Exception e) {
            log.error("执行失败", e);
        }
    }

    /**
     * @return
     * @Description 微服务文件存放路径
     * @return: java.lang.String
     */
    public static String getObjectKey() {
        // 项目或者服务名称 + 日期存储方式
        return "obs" + "/" + new SimpleDateFormat("yyyy-MM-dd").format(new Date()) + "/";
    }

4.新增业务层文件HweiYunOBSService

import com.obs.services.model.CompleteMultipartUploadResult;
import com.obs.services.model.PutObjectResult;
import com.roncoo.education.course.web.model.FileUploadStatus;
import org.springframework.web.multipart.MultipartFile;

import java.io.InputStream;
import java.util.List;

/**
 * @Description 华为云OBS服务接口
 */
public interface HweiYunOBSService {

    /**
     * @Description 删除文件
     * @param: objectKey 文件名
     * @return: boolean 执行结果
     */
    boolean delete(String objectKey);

    /**
     * @Description 批量删除文件
     * @param: objectKeys 文件名集合
     * @return: boolean 执行结果
     */
    boolean delete(List<String> objectKeys);

    /**
     * @Description 上传文件
     * @param: uploadFile 上传文件
     * @param: objectKey 文件名称
     * @return: java.lang.String url访问路径
     */
    PutObjectResult fileUpload(MultipartFile uploadFile, String objectKey);

    /**
     * 并发分片上传
     *
     * @param uploadFile
     * @param objectKey
     * @return
     */
    CompleteMultipartUploadResult splitUpload(MultipartFile uploadFile, String objectKey);

    /**
     * 获取上传速度
     *
     * @param objectKey
     * @return
     */
    public FileUploadStatus getFileUploadPlan(String objectKey);

    /**
     * @Description 文件下载
     */
    InputStream fileDownload(String objectKey);
}

5.业务层实现类文件HweiYunOBSServiceImpl

import com.obs.services.ObsClient;
import com.obs.services.exception.ObsException;
import com.obs.services.model.*;
import com.roncoo.education.course.web.model.FileUploadStatus;
import com.roncoo.education.course.web.model.HweiOBSConfig;
import com.roncoo.education.course.web.service.HweiYunOBSService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * @ClassName: HweiYunOBSServiceImpl
 * @Description: 华为云OBS服务业务层
 */
@Slf4j
@Service
public class HweiYunOBSServiceImpl implements HweiYunOBSService {

    @Autowired
    private HweiOBSConfig hweiOBSConfig;
    // 文件标签
    @Override
    public boolean delete(String objectKey) {
        ObsClient obsClient = null;
        try {
            // 创建ObsClient实例
            obsClient = hweiOBSConfig.getInstance();
            // obs删除
            obsClient.deleteObject(hweiOBSConfig.getBucketName(), objectKey);
        } catch (ObsException e) {
            log.error("obs删除保存失败", e);
        } finally {
            hweiOBSConfig.destroy(obsClient);
        }
        return true;
    }

    @Override
    public boolean delete(List<String> objectKeys) {
        ObsClient obsClient = null;
        try {
            obsClient = hweiOBSConfig.getInstance();
            DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(hweiOBSConfig.getBucketName());
            objectKeys.forEach(deleteObjectsRequest::addKeyAndVersion);
            // 批量删除请求
            obsClient.deleteObjects(deleteObjectsRequest);
            return true;
        } catch (ObsException e) {
            log.error("obs删除保存失败", e);
        } finally {
            hweiOBSConfig.destroy(obsClient);
        }
        return false;
    }

    @Override
    public PutObjectResult fileUpload(MultipartFile uploadFile, String objectKey) {
        ObsClient obsClient = null;
        try {
            String bucketName = hweiOBSConfig.getBucketName();
            // 创建实例
            obsClient = hweiOBSConfig.getInstance();
            // 判断桶是否存在
            boolean exists = obsClient.headBucket(bucketName);
            if (!exists) {
                // 若不存在，则创建桶
                HeaderResponse response = obsClient.createBucket(bucketName);
                log.info("创建桶成功" + response.getRequestId());
            }
            // 获取文件信息
            InputStream inputStream = uploadFile.getInputStream();
            long available = inputStream.available();
            if (available == 0) {
                throw new IllegalArgumentException("No data in the input stream");
            }
            PutObjectRequest request = new PutObjectRequest(bucketName, objectKey, inputStream);
            ObjectMetadata objectMetadata = new ObjectMetadata();
            objectMetadata.setContentLength(available);
            request.setMetadata(objectMetadata);
            // 设置对象访问权限为公共读
            request.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
            PutObjectResult result = obsClient.putObject(request);
            // 读取该已上传对象的URL
            log.info("已上传对象的URL" + result.getObjectUrl());
            return result;
        } catch (ObsException e) {
            log.error("obs上传失败", e);
        } catch (IOException e) {
            log.error("上传失败", e);
        } finally {
            hweiOBSConfig.destroy(obsClient);
        }
        return null;
    }

    @Override
    public CompleteMultipartUploadResult splitUpload(MultipartFile uploadFile, String objectKey) {
        // 初始化线程池
        ExecutorService executorService = Executors.newFixedThreadPool(5);
        List<PartEtag> partETags = Collections.synchronizedList(new ArrayList<>());
        ObsClient obsClient = null;
        CompleteMultipartUploadResult completeMultipartUploadResult = null;
        try {
            String bucketName = hweiOBSConfig.getBucketName();
            // 创建实例
            obsClient = hweiOBSConfig.getInstance();
            // 判断桶是否存在
            boolean exists = obsClient.headBucket(bucketName);
            if (!exists) {
                // 若不存在，则创建桶
                HeaderResponse response = obsClient.createBucket(bucketName);
                log.info("创建桶成功" + response.getRequestId());
            }
            File sampleFile = transferToFile(uploadFile);
            // 初始化分段上传任务
            InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectKey);
            // 设置对象访问权限为公共读
            request.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
            InitiateMultipartUploadResult result = obsClient.initiateMultipartUpload(request);
            String uploadId = result.getUploadId();
            log.info("\t" + "上传uploadId" + uploadId + "\n");
            // 每段上传10MB
            long partSize = 10 * 1024 * 1024L;
            long fileSize = sampleFile.length();
            // 计算需要上传的段数
            long partCount = fileSize % partSize == 0 ? fileSize / partSize : fileSize / partSize + 1;
            // 执行并发上传段
            for (int i = 0; i < partCount; i++) {
                // 分段在文件中的起始位置
                long offset = i * partSize;
                // 分段大小
                long currPartSize = (i + 1 == partCount) ? fileSize - offset : partSize;
                // 分段号
                int partNumber = i + 1;
                ObsClient finalObsClient = obsClient;
                executorService.execute(() -> {
                    UploadPartRequest uploadPartRequest = new UploadPartRequest();
                    uploadPartRequest.setBucketName(bucketName);
                    uploadPartRequest.setObjectKey(objectKey);
                    uploadPartRequest.setUploadId(uploadId);
                    uploadPartRequest.setFile(sampleFile);
                    uploadPartRequest.setPartSize(currPartSize);
                    uploadPartRequest.setOffset(offset);
                    uploadPartRequest.setPartNumber(partNumber);
                    UploadPartResult uploadPartResult;
                    try {
                        uploadPartResult = finalObsClient.uploadPart(uploadPartRequest);
                        log.info("分段#" + partNumber + " 完成\n");
                        partETags.add(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber()));
                    } catch (ObsException e) {
                        e.printStackTrace();
                    }
                });
            }
            // 等待上传完成
            executorService.shutdown();
            try {
                // 等待线程池关闭，最多等待 5 秒
                if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
                    // 如果等待超时，强制关闭线程池
                    executorService.shutdownNow();
                }
            } catch (InterruptedException e) {
                // 如果等待过程中发生异常，强制关闭线程池
                executorService.shutdownNow();
            }
            while (!executorService.isTerminated()) {
                try {
                    executorService.awaitTermination(5, TimeUnit.SECONDS);
                } catch (InterruptedException e) {
                    // 如果等待过程中发生异常，强制关闭线程池
                    executorService.shutdownNow();
                }
            }
           /* if (partETags.size() != partCount) {
                throw new IllegalStateException("部分分段未完成！");
            } else {
                System.out.println("成功完成部分" + objectKey + "\n");
            }
             //查看最近上传的所有部分
            ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, objectKey, uploadId);
            ListPartsResult partListing = obsClient.listParts(listPartsRequest);
            for (Multipart part : partListing.getMultipartList()) {
                System.out.println("\t部分#" + part.getPartNumber() + ", ETag=" + part.getEtag());
            }
            //完成部分
            Collections.sort(partETags, Comparator.comparingInt(PartEtag::getPartNumber));*/
            // 合并段
            CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, objectKey, uploadId, partETags);
            completeMultipartUploadResult = obsClient.completeMultipartUpload(completeMultipartUploadRequest);
        } catch (ObsException e) {
            log.error("obs上传失败", e);
        } finally {
            try {
                if (obsClient != null) {
                    obsClient.close();
                }
            } catch (ObsException | IOException e) {
                log.error("关闭OBS客户端失败", e);
            } finally {
                hweiOBSConfig.destroy(obsClient);
            }
        }
        return completeMultipartUploadResult;
    }

    /**
     * 获取文件上传进度
     *
     * @param objectKey
     * @return
     */
    @Override
    public FileUploadStatus getFileUploadPlan(String objectKey) {
        ObsClient obsClient = null;
        FileUploadStatus fileUploadStatus = new FileUploadStatus();
        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
            obsClient = hweiOBSConfig.getInstance();
            if (obsClient == null) {
                throw new RuntimeException("Failed to get OBS client instance");
            }
            GetObjectRequest request = new GetObjectRequest(hweiOBSConfig.getBucketName(), objectKey);
            request.setProgressListener(status -> {
                // 上传的平均速度
                fileUploadStatus.setAvgSpeed(status.getAverageSpeed());
                // 上传的百分比
                fileUploadStatus.setPct(String.valueOf(status.getTransferPercentage()));
            });
            // 每下载1MB数据反馈下载进度
            request.setProgressInterval(1024 * 1024L);
            ObsObject obsObject = obsClient.getObject(request);
            // 读取对象内容
            InputStream input = obsObject.getObjectContent();
            byte[] b = new byte[1024];
            int len;
            try {
                while ((len = input.read(b)) != -1) {
                    // 将每一次的数据写入缓冲区
                    byteArrayOutputStream.write(b, 0, len);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    input.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            return fileUploadStatus;
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            hweiOBSConfig.destroy(obsClient);
        }
        return null;
    }


    @Override
    public InputStream fileDownload(String objectKey) {
        ObsClient obsClient = null;
        try {
            String bucketName = hweiOBSConfig.getBucketName();
            obsClient = hweiOBSConfig.getInstance();
            ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
            return obsObject.getObjectContent();
        } catch (ObsException e) {
            log.error("obs文件下载失败", e);
        } finally {
            hweiOBSConfig.destroy(obsClient);
        }
        return null;
    }

    private File transferToFile(MultipartFile multipartFile) {
        // 选择用缓冲区来实现这个转换即使用java 创建的临时文件 使用 MultipartFile.transferto()方法 。
        File file = null;
        try {
            String originalFilename = multipartFile.getOriginalFilename();
            String[] filename = originalFilename.split("\\.");
            file = File.createTempFile(filename[0], filename[1]);
            multipartFile.transferTo(file);
            file.deleteOnExit();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return file;
    }

}

以上全都集成完毕，然后就可以在项目引入进行接口调用了。