使用线程池批量下载文件

使用线程池的优点：

减少资源创建 => 减少内存开销，创建线程占用内存
降低系统开销 => 创建线程需要时间，会延迟处理的请求
提高稳定性 => 避免无限创建线程引起的OutOfMemoryError【简称OOM】

Executors有以下四个方法可以创建线程池:

Executors.newFixedThreadPool() => 创建一个装有固定数量线程的线程池
Executors.newSingleThreadExecutor() => 创建单线程的线程池
Executors.newCachedThreadPool() => 让线程池根据处理需求动态地调整线程数量，比如线程池内线程的数量超过需求时，自动回收线程
Executors.newScheduledThreadPool() => 创建一个定时、周期性执行任务的线程池

ThreadPoolExecutor的构造函数共有四个，但最终调用的都是同一个：

public ThreadPoolExecutor(int corePoolSize,
                          int maximumPoolSize,
                          long keepAliveTime,
                          TimeUnit unit,
                          BlockingQueue<Runnable> workQueue,
                          ThreadFactory threadFactory,
                          RejectedExecutionHandler handler)

具体参数说明：
                          
corePoolSize => 线程池核心线程数量

maximumPoolSize => 线程池最大数量

keepAliveTime => 空闲线程存活时间

unit => 时间单位

workQueue => 线程池所使用的缓冲队列

threadFactory => 线程池创建线程使用的工厂

handler => 线程池对拒绝任务的处理策略

线程池执行逻辑说明：

1、判断核心线程数是否已满，核心线程数大小和corePoolSize参数有关，未满则创建线程执行任务

2、若核心线程池已满，判断队列是否满，队列是否满和workQueue参数有关，若未满则加入队列中

3、若队列已满，判断线程池是否已满，线程池是否已满和maximumPoolSize参数有关，若未满创建线程执行任务

4、若线程池已满，则采用拒绝策略处理无法执行的任务，拒绝策略和handler参数有关

利用多线程同步批量下载文件样例：

package com.example.demo;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

public class ThreadUtil {

    /**
     * 创建下载线程池
     *
     * @param threadSize 线程数量
     * @return ExecutorService
     */
    public static ExecutorService buildDownThreadPool(int threadSize) {
        // 空闲线程存活时间
        long keepAlive = 0L;
        // 下载线程名称前缀
        String dwonThreadNamePrefix = "dwonThread-pool";
        // 构建线程工厂
        ThreadFactory factory = ThreadUtil.buildThreadFactory(dwonThreadNamePrefix);
        //创建线程池
        return new ThreadPoolExecutor(threadSize,
                threadSize,
                keepAlive,
                TimeUnit.SECONDS,
                new ArrayBlockingQueue<>(threadSize),
                factory);
    }

    /**
     * 创建自定义线程工厂
     *
     * @param threadNamePrefix 线程名称前缀
     * @return ThreadFactory
     */
    public static ThreadFactory buildThreadFactory(String threadNamePrefix) {
        return new CustomThreadFactory(threadNamePrefix);
    }

    /**
     * 自定义线程工厂
     */
    public static class CustomThreadFactory implements ThreadFactory {
        //线程名称前缀
        private String threadNamePrefix;

        private AtomicInteger counter = new AtomicInteger(1);

        /**
         * 自定义线程工厂
         *
         * @param threadNamePrefix 线程名称前缀
         */
        CustomThreadFactory(String threadNamePrefix) {
            this.threadNamePrefix = threadNamePrefix;
        }

        @Override
        public Thread newThread(Runnable r) {
            String threadName = threadNamePrefix + "-t" + counter.getAndIncrement();
            return new Thread(r, threadName);
        }
    }
}

package com.example.demo;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;
import java.util.concurrent.*;

/**
 * 文件下载工具类
 *
 */
public class DownloadUtil {

    private static Logger logger = LoggerFactory.getLogger(DownloadUtil.class);

    /**
     * 下载线程数
     */
    private static final int DOWNLOAD_THREAD_NUM = 8;

    /**
     * 下载线程池
     */
    private static ExecutorService downloadExecutorService = ThreadUtil.buildDownThreadPool(DOWNLOAD_THREAD_NUM);

    /**
     * 单文件下载
     *
     * @param fileUrl 文件url,如:https://img3.doubanio.com//view//photo//s_ratio_poster//public//p2369390663.webp
     * @param path 存放路径,如:D:/opt/img/douban/my.webp
     */
    public static void download(String fileUrl, String path) throws Exception {
        // 判断存储文件夹是否已经存在或者创建成功
        if (!createFolderIfNotExists(path)) {
            logger.error("Can't create folder:{}", getFolder(path));
            return;
        }
        HttpURLConnection conn = null;
        InputStream in = null;
        FileOutputStream out = null;
        try {
            URL url = new URL(fileUrl);
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setConnectTimeout(10000);
            in = conn.getInputStream();
            out = new FileOutputStream(path);
            int len;
            byte[] arr = new byte[1024 * 1000];
            while (-1 != (len = in.read(arr))) {
                out.write(arr, 0, len);
            }
            out.flush();
        } catch (Exception e) {
            logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());
            throw new Exception(e.getMessage());
        } finally {
            try {
                if(null != conn){
                    conn.disconnect();
                }
                if (null != out) {
                    out.close();
                }
                if (null != in) {
                    in.close();
                }
            } catch (Exception e) {
                logger.error("Error to close stream: {}", e.getMessage());
                throw new Exception(e.getMessage());
            }
        }
    }

    /**
     * 批量多线程下载资源
     *
     * @param resourceMap 资源map, key为资源下载url,value为资源存储位置
     */
    public static void batchDownLoad(Map<String, String> resourceMap) {
        if (Objects.isNull(resourceMap) || resourceMap.isEmpty()) {
            return;
        }
        try {
            List<String> keys = new ArrayList<>(resourceMap.keySet());
            int size = keys.size();
            int pageNum = getPageNum(size);
            long startTime = System.currentTimeMillis();
            for (int index = 0; index < pageNum; index++) {
                int start = index * DOWNLOAD_THREAD_NUM;
                int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);
                //线程计数器
                final CountDownLatch latch = new CountDownLatch(last - start);
                // 获取列表子集
                List<String> urlList = keys.subList(start, last);
                for (String url : urlList) {
                    // 提交任务
                    Runnable task = new DownloadWorker(latch, url, resourceMap.get(url));
                    downloadExecutorService.submit(task);
                }
                latch.await();
            }
            long endTime = System.currentTimeMillis();
            logger.info("Download total time " + (endTime -startTime) + " millisSeconds");
        } catch (Exception e) {
            logger.error("{}", e);
        }finally {
            //关闭线程池
            if(null != downloadExecutorService){
                downloadExecutorService.shutdown();
            }
        }
    }
    /**
     * 创建文件夹,如果文件夹已经存在或者创建成功返回true
     *
     * @param path 路径
     * @return boolean
     */
    private static boolean createFolderIfNotExists(String path) {
        String folderName = getFolder(path);
        if (folderName.equals(path)) {
            return true;
        }
        File folder = new File(getFolder(path));
        if (!folder.exists()) {
            synchronized (DownloadUtil.class) {
                if (!folder.exists()) {
                    return folder.mkdirs();
                }
            }
        }
        return true;
    }

    /**
     * 获取文件夹
     *
     * @param path 文件路径
     * @return String
     */
    private static String getFolder(String path) {
        int index = path.lastIndexOf("/");
        return -1 != index ? path.substring(0, index) : path;
    }

    /**
     * 获取最后一个元素
     *
     * @param size 列表长度
     * @param index 下标
     * @return int
     */
    private static int getLastNum(int size, int index) {
        return index > size ? size : index;
    }

    /**
     * 获取划分页面数量
     *
     * @param size 列表长度
     * @return int 划分页面数量
     */
    private static int getPageNum(int size) {
        int tmp = size / DOWNLOAD_THREAD_NUM;
        return size % DOWNLOAD_THREAD_NUM == 0 ? tmp : tmp + 1;
    }

    /**
     * 下载线程
     */
    static class DownloadWorker implements Runnable {

        private CountDownLatch latch;
        private String url;
        private String path;

        DownloadWorker(CountDownLatch latch, String url, String path) {
            this.latch = latch;
            this.url = url;
            this.path = path;
        }

        @Override
        public void run() {
            try {
                logger.debug("Start batch:[{}] into: [{}]", url, path);
                DownloadUtil.download(url, path);
                logger.debug("Download:[{}] into: [{}] is done", url, path);
            } catch (Exception e) {
            }finally {
                latch.countDown();
            }
        }
    }

    public static void main(String[] args) {
        Map<String, String> map = new HashMap<>();
        map.put("http://jspro-test.oss-cn-shenzhen.aliyuncs.com/8d30007c83084e4bb341f73b6af05ec35470?Expires=1651224353&OSSAccessKeyId=LTAI4FvrfndXDm83BSgCrSoZ&Signature=fieoMu8Biye8KL12eh%2FgwTRus2Q%3D","D:/test/11/1.MP4");
        map.put("http://jspro-test.oss-cn-shenzhen.aliyuncs.com/179299140060451eae4b1f42ba1449ba1730?Expires=1651224414&OSSAccessKeyId=LTAI4FvrfndXDm83BSgCrSoZ&Signature=OMmBBY1AZBSDG3i9br41ndUMaX4%3D","D:/test/11/2.MP4");
        map.put("http://jspro-test.oss-cn-shenzhen.aliyuncs.com/e7e630c9ac0a4ebb89f778ab99a2a7287510?Expires=1651224485&OSSAccessKeyId=LTAI4FvrfndXDm83BSgCrSoZ&Signature=b%2Bdrk8J8V%2BcCWoPPRakCcErrO5k%3D","D:/test/11/3.MP4");
        DownloadUtil.batchDownLoad(map);
    }
}