报表下载工具

咘噜biu

已于 2023-12-08 16:55:12 修改

阅读量604

点赞数

文章标签： windows 大数据

于 2023-07-25 16:08:54 首次发布

本文链接：https://blog.csdn.net/xhom_w/article/details/131919783

版权

1.需求说明

我有一堆文件的Url地址，现在需要按照企业，项目和报表类型分类下载到对应的文件夹中

2.相关实体类

企业文件夹定义

package com.vz.utils.report;

import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

/**
 * @author visy.wang
 * @description: 企业文件夹
 * @date 2023/7/19 16:49
 */
@Data
public class EnterpriseDto {
    /**
     * 企业名称（文件夹名）
     */
    private String name;
    /**
     * 项目文件夹列表
     */
    private List<ProjectDto> projectList;

    /**
     * 添加项目
     */
    public void addProject(ProjectDto project){
        if(Objects.isNull(projectList)){
            projectList = new ArrayList<>();
        }
        projectList.add(project);
    }
}

项目文件夹定义

package com.vz.utils.report;

import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

/**
 * @author visy.wang
 * @description: 项目文件夹
 * @date 2023/7/19 16:50
 */
@Data
public class ProjectDto {
    /**
     * 项目名称（文件夹名）
     */
    private String name;

    /**
     * 报表文件夹列表
     */
    private List<ReportDto> reportList;

    /**
     * 添加报表快捷方式
     * @param reportType 数据类型
     * @param urlList 下载Url列表
     */
    public void addReport(ReportTypeEnum reportType, List<String> urlList){
        if(Objects.isNull(reportList)){
            reportList = new ArrayList<>();
        }

        ReportDto reportDto = new ReportDto();
        reportDto.setReportType(reportType);
        reportDto.setUrlList(urlList);
        reportList.add(reportDto);
    }
}

报表文件夹定义

package com.vz.utils.report;

import lombok.Data;
import java.util.List;

/**
 * @author visy.wang
 * @description: 报表文件夹
 * @date 2023/7/19 17:23
 */
@Data
public class ReportDto {
    /**
     * 报表类型（文件夹名）
     */
    private ReportTypeEnum reportType;
    /**
     * 文件下载地址列表
     */
    private List<String> urlList;
}

报表类型枚举

package com.vz.utils.report;

import lombok.AllArgsConstructor;
import lombok.Getter;

/**
 * @author visy.wang
 * @description: 报表类型枚举
 * @date 2023/7/19 16:57
 */
@Getter
@AllArgsConstructor
public enum ReportTypeEnum {
    SETTLEMENT("结算单", "结算单"),
    BI("BI", "BI"),
    DATA_REPORT("数据报表", "数据报表"),
    CONTRACT("合同", "合同");

    //文件夹名
    private final String folder;
    //类型描述
    private final String desc;
}

3.核心下载类

package com.vz.utils.report;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.net.URLDecoder;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.server.utils.mail.MailUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.*;

/**
 * @author visy.wang
 * @description: 下载工具
 * @date 2023/7/19 16:47
 */
@Slf4j
public class DownUtil {
    private static final Map<String,Boolean> folderCreateFlag = new HashMap<>();
    private static final String rootPath = "/usr/temp_dir/data_report"; //Linux文件存放根路径
    //private static final String rootPath = "E:\\test\\down"; //测试用，Windows系统
    // 创建线程池
    private static final ExecutorService pool;
    static {
        ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("report-down-pool-%d").build();
        pool = new ThreadPoolExecutor(20, 100, 5000L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<>(5000), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());
    }

    /**
     * 下载
     * @param enterpriseList 企业列表
     * @param name 下载类型 (根文件夹名)
     */
    public static void download(List<EnterpriseDto> enterpriseList, String name){
        //创建基础文件夹
        String baseFolder = rootPath + File.separator + name;
        createFolder(baseFolder);

        if(CollectionUtils.isEmpty(enterpriseList)){
            log.info("企业列表为空，下载结束！！！");
            return;
        }

        int downSuccessCount = 0, downFailureCount = 0, downRepeatCount = 0;
        long startTime = System.currentTimeMillis();
        List<CompletableFuture<Boolean>> futureList = new ArrayList<>();

        //遍历企业并创建文件夹
        for(EnterpriseDto enterprise: enterpriseList) {
            //创建企业文件夹
            String enterpriseFolder = baseFolder + File.separator + enterprise.getName();
            createFolder(enterpriseFolder);

            //处理项目列表
            List<ProjectDto> projectList = enterprise.getProjectList();
            if(CollectionUtils.isEmpty(projectList)){
                log.info("项目列表为空，文件夹：{}", enterpriseFolder);
                continue;
            }

            //遍历项目并创建文件夹
            for (ProjectDto project : projectList) {
                //创建项目文件夹
                String projectFolder = enterpriseFolder + File.separator + project.getName();
                createFolder(projectFolder);

                //处理数据列表
                List<ReportDto> reportList = project.getReportList();
                if(CollectionUtils.isEmpty(reportList)){
                    log.info("数据列表为空，文件夹：{}", projectFolder);
                    continue;
                }

                //遍历数据并创建文件夹
                for (ReportDto report : reportList) {
                    ReportTypeEnum reportType = report.getReportType();
                    //创建报表文件夹
                    String reportFolder = projectFolder + File.separator + reportType.getFolder();
                    createFolder(reportFolder);

                    List<String> urlList = report.getUrlList();
                    if(CollectionUtils.isEmpty(urlList)){
                        log.info("文件URL列表为空，文件夹：{}", reportFolder);
                        continue;
                    }

                    for (String url : urlList) {
                        //下载文件到当前文件夹
                        if(!StringUtils.hasText(url)){
                            log.info("{}下载失败，URL为空", reportType.getDesc());
                            downFailureCount ++;
                            continue;
                        }

                        log.info("正在下载{}，URL：{}", reportType.getDesc(), url);

                        try{
                            String uri = URLDecoder.decode(url.trim(), StandardCharsets.UTF_8);
                            //从Url提取文件名
                            int index = uri.lastIndexOf("/");
                            String fileName = index==-1 ? uri : uri.substring(index+1);

                            //检查是否下载过
                            String filePath = reportFolder + File.separator + fileName;
                            if(new File(filePath).exists()){
                                log.info("已下载过，无需再次下载，URL:{}", url);
                                downRepeatCount ++;
                                continue;
                            }

                            //下载并保存
                            CompletableFuture<Boolean> future = CompletableFuture.supplyAsync(() -> {
                                return downFile(reportType, url, filePath);
                            }, pool);
                            futureList.add(future);
                        }catch (Exception e){
                            downFailureCount ++;
                            log.info("{}下载失败，error:{}，URL：{}", reportType.getDesc(), url, e.getMessage(), e);
                        }
                    }
                }
            }
        }

        for (CompletableFuture<Boolean> future: futureList) {
            try {
                if(future.get()){
                    downSuccessCount ++;
                }else{
                    downFailureCount ++;
                }
            }catch (Exception e){
                downFailureCount ++;
                e.printStackTrace();
            }
        }

        long spendTime = System.currentTimeMillis() - startTime;
        log.info("【{}】全部下载完成，下载成功数：{}，下载失败数：{}，下载重复数：{}，总耗时：{}ms",
                name, downSuccessCount, downFailureCount, downRepeatCount, spendTime);
    }

    /**
     * 下载文件并保存在服务器
     * @param reportType 报表类型
     * @param url 下载地址
     * @param filePath 本地保存路径（全路径，含文件名和后缀）
     * @return 是否成功
     */
    private static boolean downFile(ReportTypeEnum reportType, String url, String filePath){
        try(HttpResponse response = HttpUtil.createGet(url).execute()){
            if(response.getStatus() == HttpStatus.OK.value()){
                //写入本地文件
                FileUtil.writeBytes(response.bodyBytes(), filePath);
                return true;
            }else{
                throw new RuntimeException("HttpStatus="+response.getStatus());
            }
        }catch (Exception e){
            log.info("{}下载失败，Error：{}，URL：{}", reportType.getDesc(), e.getMessage(), url, e);
            return false;
        }
    }

    //创建文件夹（如果不存在的话）
    private static void createFolder(String folder){
        if(Boolean.TRUE.equals(folderCreateFlag.get(folder))){
            return;
        }
        File f = new File(folder);
        boolean exists = f.exists();
        if(!exists){
            exists = f.mkdirs();
            log.info("文件夹[{}]已创建！", folder);
        }
        if(exists){
            folderCreateFlag.put(folder, Boolean.TRUE);
        }
    }
}

4.测试

package com.vz.utils.report;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

/**
 * @author visy.wang
 * @description:
 * @date 2023/7/25 16:02
 */
public class DownTest {
    public static void main(String[] args) {
        String url1 = "";
        String url2 = "";
        String url3 = "";
        String url4 = "";
        String url5 = "";
        String url6 = "";
        String url7 = "";
        String url8 = "";
        String url9 = "";
        String url10 = "";

        List<String> urlList1 = Arrays.asList(url1, url2, url3);
        List<String> urlList2 = Arrays.asList(url4, url5, url6);
        List<String> urlList3 = Arrays.asList(url7, url8, url9, url10);

        ProjectDto project1 = new ProjectDto();
        project1.setName("项目1");
        project1.addReport(ReportTypeEnum.SETTLEMENT, urlList1);
        project1.addReport(ReportTypeEnum.BI, urlList2);

        ProjectDto project2 = new ProjectDto();
        project2.setName("项目2");
        project2.addReport(ReportTypeEnum.DATA_REPORT, urlList3);

        EnterpriseDto enterprise = new EnterpriseDto();
        enterprise.setName("企业1");
        enterprise.addProject(project1);
        enterprise.addProject(project2);

        DownUtil.download(Collections.singletonList(enterprise), "第一批次");
        System.out.println("Finished");
    }
}