多线程上传和下载大文件（当文件服务器对上传文件大小有限制）

public class UploadAndDownloadBigFileUtil {

    private static final Logger LOGGER = LoggerFactory.getLogger(UploadAndDownloadBigFileUtil.class);

    private static final int SIZE = 16 * 1024;

    private static final int SLIDE_SIZE = 25 * 1024 * 1024;

    public static List<String> uploadBigFile(MultipartFile file) {

//        long startTime = System.currentTimeMillis();
        InputStream fs = null;
        List<String> list = new ArrayList<>();
        byte[] bMax = new byte[0];
        //RemoteClient client = RemoteClientFactory.getInstance(RemoteType.HESSIAN);
        try {
            fs = file.getInputStream();
            byte[] b = new byte[SIZE];
            ThreadPoolExecutor tpe = new ThreadPoolExecutor(5, 10, 0,
                    TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>(),
                    new ThreadPoolExecutor.CallerRunsPolicy());
            //存储线程的返回值
            List<Future<String>> results = new LinkedList<Future<String>>();
            int len = 0, i = 0, count = 0;
            while ((len = fs.read(b)) != -1) {
                byte[] b2 = Arrays.copyOfRange(b, 0, len);
                bMax = byteMerger(bMax, b2);
                if (++i == SLIDE_SIZE / SIZE) {
                    count++;
                    FileUploadUtil task = new FileUploadUtil((count) + file.getOriginalFilename(), bMax, count);
                    Future<String> result = tpe.submit(task);
                    bMax = new byte[0];
                    i = 0;
                    results.add(result);
                }
            }
            count++;
            FileUploadUtil task = new FileUploadUtil((count) + file.getOriginalFilename(), bMax, count);
            Future<String> result = tpe.submit(task);
            results.add(result);
            tpe.shutdown();
            try {
                while (!tpe.awaitTermination(1, TimeUnit.SECONDS)) {
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            for (int j = 0; j < count; j++) {
                try {
                    list.add(results.get(j).get());
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    e.printStackTrace();
                }
            }

//            System.out.println("总上传耗时：" + (System.currentTimeMillis() - startTime) / 1000 + "ms");
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (fs == null) {
                try {
                    fs.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return list;
    }

    public static void downloadBigFile(HttpServletRequest request, HttpServletResponse response, String fileName, List<String> urlList) {
        OutputStream fos = null;
        InputStream inputStream = null;
        //String data=sysAttachmentEntity.getPartUrls();
        try {
            /*List<String> list = new ArrayList<>();
            for (String s1 : data.split(",")) {
                list.add(s1);
            }*/
            //String preUrl = udfsUrl;
            BufferedOutputStream output = null;
//            System.out.println(urlList);
            //设置ContentType 和 Header
            //获得浏览器信息并转换为大写
            String agent = request.getHeader("User-Agent").toUpperCase();
            //IE浏览器和Edge浏览器
            if (agent.indexOf("MSIE") > 0 || agent.indexOf("EDGE")>0 ||
                agent.indexOf("RV:11")>0){
                //处理空格转为加号的问题
                response.setHeader("Content-Disposition", "attachment; filename=" + URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+","%20"));
            } else {
                response.setHeader("Content-Disposition", "attachment; filename=\"" + new String(fileName.getBytes("UTF-8"),"ISO8859-1")+"\"");
            }
            response.setContentType("application/octet-stream; charset=UTF-8");
            fos = response.getOutputStream();
            for (String preUrl : urlList) {
                URL url = new URL(preUrl);
                HttpURLConnection conn = (HttpURLConnection) url.openConnection();
                inputStream = conn.getInputStream();
                byte[] b = new byte[1024];
                int len = 0;
                while ((len = inputStream.read(b)) != -1) {
                    fos.write(b, 0, len);
                    fos.flush();
                }
                inputStream.close();
            }
            fos.close();
//            Enumeration<InputStream> en = Collections.enumeration(al);
//            // 将多个流合成序列流
//            SequenceInputStream sis = new SequenceInputStream(en);
//            byte[] b = new byte[1024];
//            int len = 0;
//            while ((len = sis.read(b)) != -1) {
//                fos.write(b, 0, len);
//                fos.flush();
//            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (inputStream != null) {
                try {
                    inputStream.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (fos != null) {
                try {
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

}

public class FileUploadUtil implements Callable<String> {

    private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadUtil.class);

    private String filename;
    private byte[] bytes;
    private int order;

    public FileUploadUtil(String filename, byte[] bytes, int order) {
        this.filename = filename;
        this.bytes = bytes;
        this.order = order;
    }

    @Override
    public String call() throws Exception {
        LOGGER.info("上传文件" + filename + "执行中");
        long startTime = System.currentTimeMillis();
        try {
            RemoteClient client = RemoteClientFactory.getInstance(RemoteType.HESSIAN);
            UDFSUploadVO vo = new UDFSUploadVO();
            vo.setName(filename);
            vo.setData(bytes);
            vo.setBusinessLine(BusinessLineEnum.INNER);
            vo.setPermission(UDFSPermissionEnum.GLOBAL);
            UDFSUploadResultVO resultVO = (UDFSUploadResultVO) client.executeToObject("ucarudfs.commonResourceInsert.service", vo);
            LOGGER.info(filename+"上传耗时："+(System.currentTimeMillis()-startTime)/1000+"ms");
            String url = FileTransferUtils.getUrlPrefix() + resultVO.getOriginalName();
            if(StringUtils.isNotBlank(url)){
                return url;
            }else {
                LOGGER.error("多线程上传失败");
                throw new RuntimeException("多线程上传失败！！！");
            }
        } catch (Exception e) {
            LOGGER.error("远程调用service异常", e);
        }
        return "error" ;
    }
}

思路：将大文件切分成25M每块依次上传，创建线程池利用多线程的方法不断将每块上传到文件服务器，大大地缩短了全部文件上传成功的时间，全部上传成功会返回一个文件服务器存储上传文件 url 的 list 集合；可以利用这个 list 集合去文件服务器进行文件下载

转载于:https://www.cnblogs.com/liuqing576598117/p/11102771.html