Java 多线程压缩改进

Java 多线程压缩改进

• 上一篇多线程压缩，压缩文件过多，过大的文件，会导致内存溢出，改进如下
  • 提供了一个参数，用于设置多线程压缩，压缩文件大小的最大总和
  • 每一次达到最大压缩大小，就将重新启动一次多线程压缩
  • 经过多次压缩后，合并多个压缩文件吗，得到最后的结果

具体代码如下

• maven 坐标

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-compress</artifactId>
            <version>1.20</version>
        </dependency>

package top.youlingdada.http;

import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.parallel.InputStreamSupplier;
import org.apache.commons.io.input.NullInputStream;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.*;
import java.util.zip.*;

/**
 * @author youlingdada youlingdada@163.com
 * @version 1.0
 * @createDate 2022/1/30 11:15
 */
public class ZipCompressUtils {
    private static final ThreadLocal<Long> maxSize = new ThreadLocal<>();
    private static final ThreadLocal<Long> currentSize = new ThreadLocal<>();
    private static final ThreadLocal<String> stopPath = new ThreadLocal<>();

    /**
     * 压缩文件夹
     *
     * @param zipOutName zip输出路径
     * @param paths      将要压缩的路径
     * @param max        max 应该设置大小压缩的每个文件得大小最大值，否则将造成文件无法压缩
     * @throws IOException
     * @throws ExecutionException
     * @throws InterruptedException
     */
    public static void compressFiles(String zipOutName, long max, String... paths) throws IOException, ExecutionException, InterruptedException {
        maxSize.set(max);
        stopPath.set(null);
//        创建一个线程池对象
//        压缩等级默认为速度优先
        List<String> zipFiles = new ArrayList<>();
        int n = 0;
        do {
            Path tempFile = Files.createTempFile(System.currentTimeMillis() + "", "zip");
            zipFiles.add(tempFile.toString());
            System.out.println("重载压缩次数:" + n++);
            currentSize.set(max);
            // 需要每次创建，apache commons 包内的多线程压缩会将线程池停掉，这里未确认，带改进
            ExecutorService executor = new ThreadPoolExecutor(5, 10, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(20), Executors.defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy());
            compressFiles(tempFile.toString(), executor, Deflater.BEST_SPEED, paths);
        } while (stopPath.get() != null);
        // 合并所有压缩包
        marge(zipOutName, zipFiles);
        // 删除临时文件
        for (String str : zipFiles) {
            new File(str).delete();
        }
    }

    /**
     * 自定义线程池
     *
     * @param zipOutName
     * @param executorService 线程池实现对象
     * @param paths
     * @throws IOException
     * @throws ExecutionException
     * @throws InterruptedException
     */
    public static void compressFiles(String zipOutName, ExecutorService executorService, int level, String... paths) throws IOException, ExecutionException, InterruptedException {
//      创建用于多线程压缩文件的对象
        ParallelScatterZipCreator parallelScatterZipCreator = new ParallelScatterZipCreator(executorService);
//        输出文件流
        OutputStream outputStream = Files.newOutputStream(Paths.get(zipOutName));
//        输出Zip文件流
        ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
//        设置压缩等级
        zipArchiveOutputStream.setLevel(level);
//        设置压缩的字符编码
        zipArchiveOutputStream.setEncoding("UTF-8");
//        循环压缩各个路径的文件
        for (String path : paths) {
            File temp = new File(path);
            compress(parallelScatterZipCreator, temp, temp.getName());
        }
//        将数据写入zip输出流
        parallelScatterZipCreator.writeTo(zipArchiveOutputStream);
//        相关流的关闭
        zipArchiveOutputStream.close();
        outputStream.close();
    }

    /**
     * 自定义线程创建工厂
     *
     * @param zipOutName
     * @param factory    线程创建工厂
     * @param level      压缩等级
     * @param paths
     * @throws IOException
     * @throws ExecutionException
     * @throws InterruptedException
     */
    public static void compressFiles(String zipOutName, ThreadFactory factory, int level, String... paths) throws IOException, ExecutionException, InterruptedException {
        ExecutorService executor = new ThreadPoolExecutor(5, 10, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(20), factory, new ThreadPoolExecutor.CallerRunsPolicy());
        compressFiles(zipOutName, executor, level, paths);
    }


    /**
     * 遍历压缩
     *
     * @param parallelScatterZipCreator 线程池压缩对象
     * @param inputFile                 将要压缩的文件路径,绝对路径
     * @param relativePath              相对与压缩包内的路径
     * @throws IOException
     * @throws ExecutionException
     * @throws InterruptedException
     */
    protected static void compress(ParallelScatterZipCreator parallelScatterZipCreator, File inputFile, String relativePath) throws IOException, ExecutionException, InterruptedException {
//        文件流为空，返回
        if (inputFile == null) {
            return;
        }
//        文件为文件夹，递归遍历文件
        if (inputFile.isDirectory()) {
//            获取文件内的所有文件
            File[] files = inputFile.listFiles();
            if (files == null) {
                return;
            }
//            遍历处理文件
            for (File file : files) {
                if (file.isDirectory()) {
                    compress(parallelScatterZipCreator, new File(inputFile.getAbsolutePath() + "/" + file.getName()), relativePath + "/" + file.getName());
                } else {
//                    转化为InputStreamSupplier对象
                    final InputStreamSupplier inputStreamSupplier = () -> {
                        try {
                            return new FileInputStream(file);
                        } catch (FileNotFoundException e) {
                            e.printStackTrace();
                            return new NullInputStream(0);
                        }
                    };
                    String stop = stopPath.get();
                    String path = relativePath + "/" + file.getName();
                    if (stop != null && !stop.equals(path)) {
                        continue;
                    }
                    stopPath.set(null);

                    if (currentSize.get() < file.length()) {
                        if (maxSize.get() < file.length()) {
                            throw new IllegalArgumentException("设置最大压缩容量应大于每个文件的大小,当前容量大小：" + maxSize.get() + ",文件大小：" + file.length());
                        }
                        stopPath.set(path);
                        break;
                    } else {
                        currentSize.set(currentSize.get() - file.length());
                    }
//                    添加ZipArchiveEntity对象，这里的构造函数的值，name属性，是相对于zip文件内的路径
                    ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(path);
//                    设置压缩算法
                    zipArchiveEntry.setMethod(ZipArchiveEntry.DEFLATED);
//                    设置未压缩文件的大小
                    zipArchiveEntry.setSize(file.length());
//                    添加添加ZipArchiveEntity对象到多线程压缩中
                    parallelScatterZipCreator.addArchiveEntry(zipArchiveEntry, inputStreamSupplier);
                }
            }
        } else {
            String stop = stopPath.get();
            String path = relativePath + "/" + inputFile.getName();
            if (stop != null && !stop.equals(path)) {
                return;
            }
            stopPath.set(null);

            if (currentSize.get() < inputFile.length()) {
                if (maxSize.get() < inputFile.length()) {
                    throw new IllegalArgumentException("设置最大压缩容量应大于每个文件的容量,当前容量大小：" + maxSize.get() + ",文件大小：" + inputFile.length());
                }
                stopPath.set(path);
                return;
            } else {
                currentSize.set(currentSize.get() - inputFile.length());
            }
//            当是文件时，直接处理
            final InputStreamSupplier inputStreamSupplier = () -> {
                try {
                    return new FileInputStream(inputFile);
                } catch (FileNotFoundException e) {
                    e.printStackTrace();
                    return new NullInputStream(0);
                }
            };
            ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(relativePath + "/" + inputFile.getName());
            zipArchiveEntry.setMethod(ZipArchiveEntry.DEFLATED);
            zipArchiveEntry.setSize(inputFile.length());
            parallelScatterZipCreator.addArchiveEntry(zipArchiveEntry, inputStreamSupplier);
        }
    }

    /**
     * 合并压缩文件
     *
     * @param output 合并后的文件
     */
    public static void marge(String output, List<String> sourceZipFiles) {
        byte buff[] = new byte[8192]; // 设置缓冲区大小
        ZipOutputStream out = null;
        List<ZipInputStream> ins = new ArrayList<>();
        try {
            out = new ZipOutputStream(Files.newOutputStream(Paths.get(output)));
            HashSet<String> names = new HashSet<>();
            for (String sourceZipFile : sourceZipFiles) {

                ZipFile zipFile = new ZipFile(sourceZipFile, StandardCharsets.UTF_8);
                ZipInputStream zipInputStream = new ZipInputStream(Files.newInputStream(Paths.get(sourceZipFile)));
                ins.add(zipInputStream);
                ZipEntry ze;
                Enumeration<? extends ZipEntry> enumeration = zipFile.entries();
                while (enumeration.hasMoreElements()) {
                    ze = enumeration.nextElement();
                    if (!ze.isDirectory()) {
                        if (names.contains(ze.getName())) {
                            continue;
                        }
                        ZipEntry oze = new ZipEntry(ze.getName());
                        out.putNextEntry(oze);
                        if (ze.getSize() > 0) {
                            DataInputStream dis = new DataInputStream(zipFile.getInputStream(ze));
                            int len = 0;
                            while ((len = dis.read(buff)) > 0) {
                                out.write(buff, 0, len);
                            }
                            out.closeEntry();
                            out.flush();
                        }
                        names.add(oze.getName());
                    }

                }
                zipInputStream.closeEntry();
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            throw new RuntimeException("合并压缩文件失败");
        } finally {
            try {
                if (out != null) {
                    out.close();
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
            try {
                for (ZipInputStream in : ins) {
                    if (in != null) {
                        in.close();
                    }
                }
            } catch (IOException ex) {
                ex.printStackTrace();
            }
        }
    }
}