java 高性能文件压缩_Java创建100MB压缩的csv文件性能问题

您的测试数据大约为1GB，压缩到100MB。根据您的硬件，可能无法实现5秒钟的性能。

我把一个快速和肮脏的基准放在一起，突出了写入到zip文件的性能影响。

写入CSV与String.join()：9.6s

写入CSV具有超强CSV：12.7s

写入CSV拉链内的String.join()：18.6s

写入CSV具有超强CSV拉链内：22.5s

看起来，使用Super CSV(〜122％)有一点点开销，但只需写入一个zip文件几乎会使时间增加一倍(〜190％)，而不管是否Supe使用CSV。

下面是4个场景的代码。与您提供的代码不同，我直接写入一个文件(我没有注意到写入磁盘与写入内存之间的任何差异，即ByteArrayOutputStream)。我也跳过了关于超级CSV示例的BufferedWriter，因为它已经在内部使用了它，并且我使用了试用资源来使事情更加清洁。

@Test

public void testWriteToCsvFileWithSuperCSV() throws Exception {

long startTime = System.currentTimeMillis();

try (FileOutputStream csvFile = new FileOutputStream(new File("supercsv.csv"));

ICsvListWriter writer = new CsvListWriter(new OutputStreamWriter(csvFile, "UTF-8"), CsvPreference.EXCEL_PREFERENCE)

){

for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) {

final List rowContent = new LinkedList<>();

for (int colIdx = 0; colIdx < 6; colIdx++) {

String str = "R" + rowIdx + "C" + colIdx + " FieldContent";

rowContent.add(str);

}

writer.write(rowContent);

}

}

long stopTime = System.currentTimeMillis();

long elapsedTime = stopTime - startTime;

System.out.println("Writing to CSV with Super CSV took " + (elapsedTime/1000f) + " seconds");

}

@Test

public void testWriteToCsvFileWithinZipWithSuperCSV() throws Exception {

long startTime = System.currentTimeMillis();

try (FileOutputStream zipFile = new FileOutputStream(new File("supercsv.zip"));

ZipOutputStream zos = new ZipOutputStream(zipFile);

ICsvListWriter writer = new CsvListWriter(new OutputStreamWriter(zos, "UTF-8"), CsvPreference.EXCEL_PREFERENCE)

){

ZipEntry csvFile = new ZipEntry("supercsvwithinzip.csv");

zos.putNextEntry(csvFile);

for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) {

final List rowContent = new LinkedList<>();

for (int colIdx = 0; colIdx < 6; colIdx++) {

String str = "R" + rowIdx + "C" + colIdx + " FieldContent";

rowContent.add(str);

}

writer.write(rowContent);

}

}

long stopTime = System.currentTimeMillis();

long elapsedTime = stopTime - startTime;

System.out.println("Writing to CSV within zip file with Super CSV took " + (elapsedTime/1000f) + " seconds");

}

@Test

public void testWriteToCsvFileWithStringJoin() throws Exception {

long startTime = System.currentTimeMillis();

try (FileOutputStream textFile = new FileOutputStream(new File("join.csv"));

BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(textFile, "UTF-8"));

){

for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) {

final List rowContent = new LinkedList<>();

for (int colIdx = 0; colIdx < 6; colIdx++) {

String str = "R" + rowIdx + "C" + colIdx + " FieldContent";

rowContent.add(str);

}

writer.append(String.join(",", rowContent) + "\n");

}

}

long stopTime = System.currentTimeMillis();

long elapsedTime = stopTime - startTime;

System.out.println("Writing to CSV with String.join() took " + (elapsedTime/1000f) + " seconds");

}

@Test

public void testWriteToCsvFileWithinZipWithStringJoin() throws Exception {

long startTime = System.currentTimeMillis();

try (FileOutputStream zipFile = new FileOutputStream(new File("join.zip"));

ZipOutputStream zos = new ZipOutputStream(zipFile);

BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zos, "UTF-8"));

){

ZipEntry csvFile = new ZipEntry("joinwithinzip.csv");

zos.putNextEntry(csvFile);

for (int rowIdx = 0; rowIdx < 7000000; rowIdx++) {

final List rowContent = new LinkedList<>();

for (int colIdx = 0; colIdx < 6; colIdx++) {

String str = "R" + rowIdx + "C" + colIdx + " FieldContent";

rowContent.add(str);

}

writer.append(String.join(",", rowContent) + "\n");

}

}

long stopTime = System.currentTimeMillis();

long elapsedTime = stopTime - startTime;

System.out.println("Writing to CSV within zip with String.join() took " + (elapsedTime/1000f) + " seconds");

}