问题说明:有一个无续long型的大文件,超过系统内存,需要对其进行排序。
大文件排序,多线程并发处理:
import lombok.SneakyThrows;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 大文件排序
*/
public class BigFileSort {
private static final Random RANDOM = new Random();
private long splitLength = 10;
private AtomicInteger atomicInteger = new AtomicInteger();
private int totalThread = 0;
public BigFileSort(long splitLength) {
this.splitLength = splitLength;
}
/**
* 创建大文件
*/
public void createBigFile(String fileName, long amount) throws IOException {
FileOutputStream out = null;
try {
File file = new File(fileName);
if (file.exists()) {
if (file.isDirectory()) {
throw new IOException("File '" + file + "' exists but is a directory");
}
if (!file.canWrite()) {
throw new IOException("File '" + file + "' cannot be written to");
}
} else {
File parent = file.getParentFile();
if (parent != null && !parent.exists() && !parent.mkdirs()) {
throw new IOException("File '" + file + "' could not be created");
}
}
out = new FileOutputStream(file);
for (long i = 0; i < amount; i ++) {
long value = RANDOM.nextLong();
value = value < 0 ? -value : value;
out.write(String.valueOf(value + " ").getBytes());
}
} finally {
IOUtils.closeQuietly(out);
}
}
/**
* 拆分小文件
*/
public void splitFile(String fileName) throws IOException {
FileInputStream fis = null;
FileOutputStream out = null;
try {
File file = new File(fileName);
fis = new FileInputStream(file);
int temp = 0;
//当temp等于-1时,表示已经到了文件结尾,停止读取
String fileParent = file.getParentFile().getAbsolutePa