在日常完成业务过程中,出现了要在内存中操作很大的txt的需求,文件每行作为一条数据,但是由于数据量很大,不能一次性拉入内存中操作,但是又需要使用hashmap进行去重,于是借鉴hashmap的源码,编写了简易的大txt分桶执行操作。
BufferedFactory(写入、写出流工厂类)
@Slf4j
public class BufferedFactory {
private static HashMap<Integer,BufferedWriter> bufferedWriterHashMap = new HashMap<>();
private static HashMap<Integer, BufferedReader> bufferedReaderHashMap = new HashMap<>();
private static String filePath;
public static void setFilePath(String filePath){
BufferedFactory.filePath = "D:\\临时统计\\"+filePath;
}
public static BufferedWriter getBufferedWriter(int n,String fileName){
new File(filePath).mkdir();
if (!bufferedWriterHashMap.containsKey(n)){
try {
bufferedWriterHashMap.put(n,new BufferedWriter(new FileWriter(filePath+"\\"+fileName+n+".txt")));
} catch (IOException e) {
log.error("创建写出流失败:"+filePath+n);
}
}
return bufferedWriterHashMap.get(n);
}
public static void closeBufferedWriter(){
bufferedWriterHashMap.forEach((key,value)->{
try {
value.close();
} catch (IOException e) {
log.error("第"+key+"个桶绑定的写出流关闭异常");
}
});
}
public static BufferedReader getBufferedReader(int n,String filename){
if (!bufferedReaderHashMap.containsKey(n)){
try {
bufferedReaderHashMap.put(n,new BufferedReader(new FileReader(filePath+"\\"+filename+n+".txt")));
} catch (IOException e) {
log.error("创建写出流失败:"+filePath+"\\"+filename+n+".txt");
}
}
return bufferedReaderHashMap.get(n);
}
public static void closeBufferedReader(){
bufferedReaderHashMap.forEach((key,value)->{
try {
value.close();
} catch (IOException e) {
log.error("第"+key+"个桶绑定的写入流关闭异常");
}
});
}
}
大文件分隔、批量执行桶中文件操作工具类
@Slf4j
public class BigFileSplitUtil {
public static void splitBigTxt(String filename,String bucketName,String bucketFilename,int n,int index){
try(BufferedReader bufferedReader = new BufferedReader(new FileReader(filename))) {
String zzqData;
int i = 0;
while (!StringUtils.isEmpty(zzqData = bufferedReader.readLine())){
final String[] split = zzqData.split("~\\$");
BufferedFactory.setFilePath(bucketName);
final BufferedWriter bufferedWriter = BufferedFactory.getBufferedWriter((n - 1) & (split[index].hashCode()),bucketFilename);
bufferedWriter.write(zzqData+"\n");
System.out.println("第"+ i++ +"条数据分桶成功");
}
}catch (IOException e){
log.error("分桶失败",e);
}finally {
BufferedFactory.closeBufferedWriter();
}
}
public static void bucketExecute(String bucketName,String bucketFilename,int n,BucketFileForExecute bucketFileForExecute){
try {
BufferedFactory.setFilePath(bucketName);
for (int i = 0; i < n; i++) {
bucketFileForExecute.execute(BufferedFactory.getBufferedReader(i, bucketFilename));
}
} finally {
BufferedFactory.closeBufferedReader();
}
}
}
批量执行桶中代码的函数式接口(不喜欢使用函数式接口的也可以不封装这个方法)
public interface BucketFileForExecute {
void execute(BufferedReader bufferedReader);
}