ZIP压缩速度测试

朋友反馈他们项目下载资源接口响应速度奇慢，数据大一点的30S起步，所以给出了优化建议，并测试压缩方法。测试资源大小 500M

所需依赖

<dependency>
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-compress</artifactId>
    <version>1.20</version>
</dependency>
<dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
    <version>20.0</version>
</dependency>

测试方法，其中除了多线程(方案4)速度有明显提升，其它感觉区别不大

注：方案4原计划是自己使用多线程实现， 结果发现这个网上有现成的

import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
import org.apache.commons.compress.archivers.zip.UnixStat;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.parallel.InputStreamSupplier;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.input.NullInputStream;

import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URLEncoder;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Date;
import java.util.concurrent.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

public class Test {

    public static void main(String[] args) throws Exception {
        f2();
    }

    /**
     // 方案1--原方案--单个文件复制后全部压缩  复制文件 0.6s,总时间13.1S
     */
    private static void f1() throws Exception {
        File file = new File("E:\\tempApply");
        String ysFile = "E:\\tempApply1\\";
        Date startTime = new Date();
        for(File f:file.listFiles()){
            File file1 = new File(ysFile + f.getName());
            if (!file1.exists()) {
                file1.mkdir();
            }
            if(f.isDirectory()){
                for(File f2:f.listFiles()){
                    File file2 = new File(ysFile + f.getName()+"\\"+f2.getName());
                    FileUtils.copyFile(f2,file2);
                }
            }
        }
        System.out.println("复制文件完成，时间："+(new Date().getTime()- startTime.getTime())+"ms");
        ZipUtils.generateFile(ysFile, "zip");
        System.out.println("压缩文件完成，总时间："+(new Date().getTime()- startTime.getTime())+"ms");
    }
    /**
     *     方案2--创建zip后直接塞入数据，和方案一基本一样，只是去除了文件复制 12.2s
     */
    private static void f2() throws IOException {
        Date startTime2 = new Date();
        File file = new File("E:\\tempApply");
        ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(Paths.get("E:\\tempApply2.zip"))));
        for(File f:file.listFiles()){
            if(f.isDirectory()){
                for(File f2:f.listFiles()){
                    FileInputStream bis = new FileInputStream(f2);
                    zos.putNextEntry(new ZipEntry(f.getName() +"\\"+ f2.getName()));
                    int count;
                    byte[] buf = new byte[1024];
                    while ((count = bis.read(buf)) > 0) {
                        zos.write(buf, 0, count);
                    }
                    bis.close();
                }
            }
        }
        zos.close();
        System.out.println("压缩文件完成，总时间："+(new Date().getTime()- startTime2.getTime())+"ms");
    }

    /**
     *     方案3--NIO后进行替换 11.8
     */
    private static void f3() throws IOException {
        Date startTime = new Date();
        File file = new File("E:\\tempApply");
        ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(Paths.get("E:\\tempApply3.zip"))));
        WritableByteChannel writableByteChannel = Channels.newChannel(zos);
        for(File f:file.listFiles()){
            for(File f2:f.listFiles()){
                try (FileChannel fileChannel = new FileInputStream(f2).getChannel()) {
                    zos.putNextEntry(new ZipEntry( f.getName() +"\\"+ f2.getName()));
                    fileChannel.transferTo(0, fileChannel.size(), writableByteChannel);
                }
            }
        }
        zos.close();
        writableByteChannel.close();
        System.out.println("压缩文件完成，总时间："+(new Date().getTime()- startTime.getTime())+"ms");
    }


    /**
     *  多线程 本地3s,response 6s
     **/
    public static void f4() throws Exception{
        Date startTime = new Date();
        File file = new File("E:\\tempApply");
        ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("compressFileList-pool-").build();
        ExecutorService executor = new ThreadPoolExecutor(5, 20, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), factory);
        ParallelScatterZipCreator parallelScatterZipCreator = new ParallelScatterZipCreator(executor);
//        setResponseHeaders(response, "tempApply4.zip");
//        OutputStream outputStream = response.getOutputStream();
        OutputStream outputStream = Files.newOutputStream(Paths.get("E:\\tempApply4.zip"));
        ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
        zipArchiveOutputStream.setEncoding("UTF-8");
        for(File f:file.listFiles()){
            for(File f2:f.listFiles()){
                final InputStreamSupplier inputStreamSupplier = () -> {
                    try {
                        return new FileInputStream(f2);
                    } catch (FileNotFoundException e) {
                        e.printStackTrace();
                        return new NullInputStream(0);
                    }
                };
                ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(f.getName() +"\\"+ f2.getName());
                zipArchiveEntry.setMethod(ZipArchiveEntry.DEFLATED);
                zipArchiveEntry.setSize(f2.length());
                zipArchiveEntry.setUnixMode(UnixStat.FILE_FLAG | 436);
                parallelScatterZipCreator.addArchiveEntry(zipArchiveEntry, inputStreamSupplier);
            }
        }
        parallelScatterZipCreator.writeTo(zipArchiveOutputStream);
        zipArchiveOutputStream.close();
        outputStream.flush();
        outputStream.close();
        System.out.println("压缩文件完成，总时间："+(new Date().getTime()- startTime.getTime())+"ms");
    }

    public static void setResponseHeaders(HttpServletResponse response,String fileName) throws UnsupportedEncodingException {
        response.setContentType("application/octet-stream; charset=utf-8");
        response.setCharacterEncoding("utf-8");
        //content-disposition属性名 （attachment表示以附件的方式下载;inline表示在页面内打开）
        response.setHeader("content-disposition", "attachment; filename="+ URLEncoder.encode(fileName, "utf-8"));
    }

ZipUtils工具类

import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
import org.apache.commons.compress.archivers.zip.UnixStat;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.parallel.InputStreamSupplier;
import org.apache.commons.io.input.NullInputStream;

import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.util.List;
import java.util.concurrent.*;

/**
 * @author pw
 */
public class ZipUtils {

    /**
     * 该线程池会被自动关闭
     * @return ZipCreator
     */
    private static ParallelScatterZipCreator getZipCreator(){
        ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("compressFileList-pool-").build();
        ExecutorService executor = new ThreadPoolExecutor(5, 20, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), factory);
        return new ParallelScatterZipCreator(executor);
    }

    /**
     *  多线程压缩
     **/
    public static void zipThreads(List<File> list, HttpServletResponse response) throws Exception{
        ParallelScatterZipCreator zipCreator = getZipCreator();
        OutputStream outputStream = response.getOutputStream();
        ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
        zipArchiveOutputStream.setEncoding("UTF-8");
        for(File f:list){
            compressFile(f, "", zipCreator);
        }
        zipCreator.writeTo(zipArchiveOutputStream);
        zipArchiveOutputStream.close();
        outputStream.flush();
        outputStream.close();
    }

    /**
     *  多线程压缩
     **/
    public static void zipThreads(String srcFilePath, HttpServletResponse response) throws Exception{
        ParallelScatterZipCreator zipCreator = getZipCreator();
        OutputStream outputStream = response.getOutputStream();
        ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputStream);
        zipArchiveOutputStream.setEncoding("UTF-8");
        File src = new File(srcFilePath);
        compressbyType(src, "", zipCreator);
        zipCreator.writeTo(zipArchiveOutputStream);
        zipArchiveOutputStream.close();
    }

    /**
     * 按照原路径的类型就行压缩。文件路径直接把文件压缩，
     * @param src 源文件
     * @param zipCreator 工具
     */
    private static void compressbyType(File src,String baseDir, ParallelScatterZipCreator zipCreator) {
        if (!src.exists()) {
            return;
        }
        if (src.isFile()) {
            //src是文件，调用此方法
            compressFile(src, baseDir, zipCreator);

        } else if (src.isDirectory()) {
            //src是文件夹，调用此方法
            compressDir(src,baseDir, zipCreator);

        }

    }

    /**
     * 压缩文件
     */
    private static void compressFile(File f,String baseDir, ParallelScatterZipCreator zipCreator) {
        final InputStreamSupplier inputStreamSupplier = () -> {
            try {
                return new FileInputStream(f);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
                return new NullInputStream(0);
            }
        };
        ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry( baseDir+f.getName());
        zipArchiveEntry.setMethod(ZipArchiveEntry.DEFLATED);
        zipArchiveEntry.setSize(f.length());
        zipArchiveEntry.setUnixMode(UnixStat.FILE_FLAG | 436);
        zipCreator.addArchiveEntry(zipArchiveEntry, inputStreamSupplier);
    }


    /**
     * 压缩文件夹
     */
    private static void compressDir(File dir,String baseDir, ParallelScatterZipCreator zipCreator) {
        if (!dir.exists()) {
            return;
        }
        File[] files = dir.listFiles();
        if(null==files || files.length == 0){
            compressbyType(dir,  baseDir + dir.getName() +"/",zipCreator);
            return;
        }
        for (File file : files) {
            compressbyType(file,  baseDir + dir.getName() + "/",zipCreator);
        }
    }
}