ThreadPoolExecutor - 批量下载文件

最新推荐文章于 2023-08-28 21:11:34 发布

麻哥哥

最新推荐文章于 2023-08-28 21:11:34 发布

阅读量139

点赞数 1

文章标签： java

本文链接：https://blog.csdn.net/hjkl0987/article/details/131429962

版权

该代码示例展示了如何在Java中利用ThreadPoolExecutor进行多线程处理，从文本文件读取用户资料ID，然后并发执行下载任务。每个任务使用Hutool的HttpUtil发送GET请求获取文档，并依据响应下载文件。

摘要由CSDN通过智能技术生成

背景：公司要下载全量用户资料，搞了一个线程池去批量执行，代码如下：

import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bidb.core.util.HttpUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class DownloadPicUtil {
    public static ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(6, 6, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<>(21000), new ThreadPoolExecutor.AbortPolicy());

    public static void main(String[] args) {
        DownloadPicUtil d = new DownloadPicUtil();
        d.startDownloadEducations();
    }

    public void startDownloadEducations() {
        BufferedReader reader = null;
        try {
            String paramsPath = "/Users/oojzoo/project/educations-1.txt";
            File file = new File(paramsPath);
            reader = new BufferedReader(new FileReader(file));
            String tempString = null;
            String tempPersonName = null;
            int tempEducation = 1;
            int line = 1;
            //一次读一行，读入null时文件结束
            while ((tempString = reader.readLine()) != null) {
                //把当前行号显示出来
                if (StringUtils.isNotBlank(tempString) && tempString.split("#").length == 5) {
                    String[] strings = tempString.split("#");
                    String educationDocId = strings[0];
                    String degreeDocId = strings[1];

                    if (strings[4].equals(tempPersonName)) {
                        tempEducation ++;
                    }
                    else {
                        tempPersonName = strings[4];
                        tempEducation = 1;
                    }
                    if (educationDocId != null) {
                        // 加入多线程任务执行
                        threadPoolExecutor.execute(new DownloadTask(educationDocId, strings[2] + "#0" + tempEducation + "#" + strings[4]));
                    }
                    if (degreeDocId != null) {
                        threadPoolExecutor.execute(new DownloadTask(degreeDocId, strings[2] + "#0" + tempEducation + "#" + strings[4] + "2"));
                    }
                }
                line++;
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
        }
    }

    /**
     * 直接创建的是一个静态内部类，实现 Runnable接口
     */
    public static class DownloadTask implements Runnable {
        Map<String, String> headers = new HashMap();
        private String docId;
        private String newFileName;
        private String getDocUrl = "https://xxxxxxx";
        private String path = "/Volumes/J_CENA_X64F/educations/";
        private String downloadDocUrl = "https://xxxxxx";
        public DownloadTask(String docId, String newName) {
            this.docId = docId;
            this.newFileName = newName;
            headers.put("Authorization", "xxxxxx");
        }

        @Override
        public void run() {
            if (docId == null) {
                return ;
            }

            try {
                String response = HttpUtils.sendGet(getDocUrl + docId, "UTF-8", headers);
                JSONObject parsed = JSONObject.parseObject(response);
                if (parsed.get("data") == null || ((JSONArray)parsed.get("data")).size() == 0) {
                    return;
                }
                JSONObject jsonObject = (JSONObject) ((JSONArray) parsed.get("data")).get(0);
                if (jsonObject.get("docId") == null || jsonObject.get("fileName") == null) {
                    return;
                }
                HttpUtil.downloadFile(downloadDocUrl + jsonObject.get("docId").toString(), path + newFileName + jsonObject.get("fileName").toString().substring(jsonObject.get("fileName").toString().lastIndexOf(".")));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    }
}