public class ResortByDel {
private static String BIG_FILE_NAME = "D:\\test1.txt";
private static Integer BATCH_SIZE = 100;
private static String LINE_SEPARATOR = ",";
private static String SORT_FILE_NAME = "D:\\test2.txt";
private static List separateFile() {
List fileNameList = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new FileReader(BIG_FILE_NAME))) {
int index = 0;
List batchLineList = new ArrayList<>(BATCH_SIZE);
String line;
while ((line = reader.readLine()) != null || batchLineList.size() > 0) {
if (line != null) batchLineList.add(line);
if (batchLineList.size() == BATCH_SIZE || (line == null && batchLineList.size() > 0)) {
// 内容排序
batchLineList.sort((a, b) -> a.compareTo(b));
// 写小文件
String fileName = BIG_FILE_NAME + ".tmp." + index++;
try (FileWriter tmpWriter = new FileWriter(fileName)) {
for (String val : batchLineList) {
tmpWriter.write(val + "\r\n");
}
}
fileNameList.add(fileName);
batchLineList.clear();
}
}
} catch (IOException e) {
e.printStackTrace();
}
return fileNameList;
}
private static void mergeFile(List fileNameList) {
Map map = new HashMap<>();
String preStr = null;
try (FileWriter writer = new FileWriter(SORT_FILE_NAME)) {
for (String fileName : fileNameList) {
BufferedReader tmpReader = new BufferedReader(new FileReader(fileName));
map.put(tmpReader, tmpReader.readLine());
}
while (true) {
boolean canRead = false;
Map.Entry minEntry = null;
for (Map.Entry entry : map.entrySet()) {
String value = entry.getValue();
if (value == null) {
continue;
}
// 获取当前 reader 内容最小 entry
if ((minEntry == null) || (Integer.valueOf(value)
minEntry = entry;
}
canRead = true;
}
// 当且仅当所有 reader 内容为空时，跳出循环
if (!canRead) {
break;
}
//去重
if (preStr != null && preStr.equals(minEntry.getValue())) {
minEntry.setValue(minEntry.getKey().readLine());
continue;
}
preStr = minEntry.getValue();
writer.write(minEntry.getValue() + "\r\n");
minEntry.setValue(minEntry.getKey().readLine());
}
} catch (Exception e) {
e.printStackTrace();
} finally {
// 注意关闭分片文件输入流
for (BufferedReader reader : map.keySet()) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public static void main(String[] args) throws IOException {
List separateFiles = separateFile();
mergeFile(separateFiles);
}
}
4969
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
