记录一次线程池使用案例

业务描述

上传一个大文件，大概有几百兆，上华为云官网找到SDK文档，有个大文件分段上传的功能，查看代码案例如下：
参考链接：华为云SDK官方文档.

这个文档中使用了一个固定大小的线程池，每次上传50M。
在做这个功能的时候试验了一个大小为381M的视频文件，然后视频分8段，共用时三分钟左右，StopWatch打印如下：

ms % Task name

00799 000% 上传开始
184414 100% 华为云上传开始
00025 000% 保存数据开始

控制器方法如下：

 @RequestMapping(params = "uploadFile")
    @ResponseBody
    public AjaxJson uploadFile(HttpServletRequest request, HttpServletResponse response) {
    	StopWatch stopWatch = new StopWatch();
    	stopWatch.start("上传开始");
        AjaxJson j = new AjaxJson();

        try {
           
            MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;

            String elecName = multipartRequest.getParameter("elecName");

            Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();

            for (Map.Entry<String, MultipartFile> entity : fileMap.entrySet()) {
                MultipartFile file = entity.getValue();// 获取上传文件对象
                boolean isLimit = FileUtil.checkFileSize(file.getSize(), 500, "M");
                if (!isLimit) {
                    j.setSuccess(false);
                    j.setMsg("文件不能超过500M！");
                    return j;
                }
                
                String originalFilename = file.getOriginalFilename();
                String[] split = originalFilename.split("\\.");
                String format = split[split.length - 1];
                if (!(format.equalsIgnoreCase("pdf") || format.equalsIgnoreCase("mp4"))) {
                    j.setSuccess(false);
                    j.setMsg("请上传PDF或MP4格式的文件！");
                    return j;
                }
                // 转换成File类型
                File tempFile = FileUtil.multipartFileToFile(file);
                if (format.equalsIgnoreCase("mp4")) {
                    // 校验编码格式，满足AVC(H264)的才能上传
                    Encoder encoder = new Encoder();
                    MultimediaInfo info = encoder.getInfo(tempFile);
                    VideoInfo video = info.getVideo();
                    String decoder = video.getDecoder();
                    if (!decoder.equalsIgnoreCase("H264")) {
                        j.setSuccess(false);
                        j.setMsg("文件编码格式有误！");
                        return j;
                    }
                }
                stopWatch.stop();
                stopWatch.start("华为云上传开始");

                //文件大小小于50M不需要分段上传
                //PutObjectResult doUpload = HuaweiCloudOBSUtil.doUpload(file.getInputStream(), UUID.randomUUID().toString() + "." + format);
                
                CompleteMultipartUploadResult doUpload = HuaweiCloudOBSUtil.SectionUpload(tempFile, file.getSize(), UUID.randomUUID().toString() + "." + format);
                stopWatch.stop();

                stopWatch.start("保存数据开始");
                ElectronicLibraryEntity elecLibrary = new ElectronicLibraryEntity();
                elecLibrary.setElecName(elecName);
                elecLibrary.setElecUrl(doUpload.getObjectUrl());
                elecLibrary.setObsKey(doUpload.getObjectKey());
                //stopWatch打印任务执行时间
                System.out.println(stopWatch.prettyPrint());
                j.setSuccess(true);
                j.setMsg("上传成功！");
            }
        } catch (Exception e) {
            e.printStackTrace();
            j.setSuccess(false);
            j.setMsg("系统异常，上传失败！");
        }
        return j;
    }

上传方法如下：

/**
	 * 华为云大文件分段上传
	 * @param input 文件输入流
	 * @param size 文件大小
	 * @param objectName 文件名
	 * @description 分段上传分为如下3个步骤：
			1.初始化分段上传任务（ObsClient.initiateMultipartUpload）。
			2.逐个或并行上传段（ObsClient.uploadPart）。
			3.合并段（ObsClient.completeMultipartUpload）或取消分段上传任务(ObsClient.abortMultipartUpload)。
	 * @author fenglu
	 * @return 
	 * @date 2020年7月17日 上午11:13:31
	 */
	public static CompleteMultipartUploadResult SectionUpload(File file , long size ,String objectName){
		String endPoint = ""; 
		
		final ObsClient ObsClient = new ObsClient(ak, sk, endPoint);
		
		final String bucketName = "bucketName";
		final String objectKey = objectName;
		// 创建ObsClient实例

		// 初始化线程池
		ExecutorService executorService = Executors.newFixedThreadPool(5);
		
		// 初始化分段上传任务
		InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectKey);
		InitiateMultipartUploadResult result = ObsClient.initiateMultipartUpload(request);

		final String uploadId = result.getUploadId();
		System.out.println("\t"+ uploadId + "\n");

		// 每段上传50MB
		long partSize = 50 * 1024 * 1024L;
		long fileSize = size;

		// 计算需要上传的段数
		long partCount = fileSize % partSize == 0 ? fileSize / partSize : fileSize / partSize + 1;

		final List<PartEtag> partEtags = Collections.synchronizedList(new ArrayList<PartEtag>());

		// 执行并发上传段
		for (int i = 0; i < partCount; i++)
		{
		    // 分段在文件中的起始位置
		    final long offset = i * partSize;
		    // 分段大小
		    final long currPartSize = (i + 1 == partCount) ? fileSize - offset : partSize;
		    // 分段号
		    final int partNumber = i + 1;
		    executorService.execute(new Runnable()
		    {
		        @Override
		        public void run()
		        {
		            UploadPartRequest uploadPartRequest = new UploadPartRequest();
		            uploadPartRequest.setBucketName(bucketName);
		            uploadPartRequest.setObjectKey(objectKey);
		            uploadPartRequest.setUploadId(uploadId);
		            uploadPartRequest.setFile(file);
		            uploadPartRequest.setPartSize(currPartSize);
		            uploadPartRequest.setOffset(offset);
		            uploadPartRequest.setPartNumber(partNumber);
		            
		            UploadPartResult uploadPartResult;
		            try
		            {
		                uploadPartResult = ObsClient.uploadPart(uploadPartRequest);
		                System.out.println("Part#" + partNumber + " done\n");
		                partEtags.add(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber()));
		            }
		            catch (ObsException e)
		            {
		                e.printStackTrace();
		            }
		        }
		    });
		}

		// 等待上传完成
		executorService.shutdown();
		while (!executorService.isTerminated())
		{
		    try
		    {
		        executorService.awaitTermination(5, TimeUnit.SECONDS);
		    }
		    catch (InterruptedException e)
		    {
		        e.printStackTrace();
		    }
		}
		// 合并段
		CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, objectKey, uploadId, partEtags);
		CompleteMultipartUploadResult completeMultipartUpload = ObsClient.completeMultipartUpload(completeMultipartUploadRequest);
		return completeMultipartUpload;
		
	}

流工具类

/**
 * @Description: 文件相关的工具类
 * @Author: zhangcg
 * @Date: 2020/6/22
 */
public class FileUtil {

    /**
     * MultipartFile 转为File类型
     * @param file
     * @return
     * @throws Exception
     */
    public static File multipartFileToFile(MultipartFile file) throws Exception {

        File toFile = null;
        if (file.equals("") || file.getSize() <= 0) {
            file = null;
        } else {
            InputStream ins = null;
            ins = file.getInputStream();
            toFile = new File(file.getOriginalFilename());
            inputStreamToFile(ins, toFile);
            ins.close();
        }
        return toFile;
    }

    //获取流文件
    private static void inputStreamToFile(InputStream ins, File file) {
        try {
            OutputStream os = new FileOutputStream(file);
            int bytesRead = 0;
            byte[] buffer = new byte[8192];
            while ((bytesRead = ins.read(buffer, 0, 8192)) != -1) {
                os.write(buffer, 0, bytesRead);
            }
            os.close();
            ins.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 判断文件大小，如：文件不大于100M(multipartFile.getSize(),100,"M")或(file.length(),100,"M")
     * @param len
     * @param size
     * @param unit
     * @return
     */
    public static boolean checkFileSize(Long len, int size, String unit) {
        double fileSize = 0;
        if ("B".equals(unit.toUpperCase())) {
            fileSize = (double) len;
        } else if ("K".equals(unit.toUpperCase())) {
            fileSize = (double) len / 1024;
        } else if ("M".equals(unit.toUpperCase())) {
            fileSize = (double) len / 1048576;
        } else if ("G".equals(unit.toUpperCase())) {
            fileSize = (double) len / 1073741824;
        }
        if (fileSize > size) {
            return false;
        }
        return true;
    }

}