文章标题

最新推荐文章于 2023-08-29 11:26:16 发布

叫我王小贱

最新推荐文章于 2023-08-29 11:26:16 发布

阅读量283

点赞数

分类专栏：算法 Java

Java 同时被 2 个专栏收录

4 篇文章 0 订阅

订阅专栏

算法

1 篇文章 0 订阅

订阅专栏

10亿个字符串的排序问题

10亿个字符串排序

一、问题描述

有一个大文件，里面有十亿个字符串，乱序的，要求将这些字符串以字典的顺序排好序

二、解决思路

    将大文件切割成小文件，每个小文件内归并排序；

    对所有的小文件进行归并排序——多重归并排序

三、解决方案

3.1 模拟产生10亿个随机字符

Java代码 
public static void generateDate() throws IOException {  
    BufferedWriter writer = new BufferedWriter(new FileWriter(ORIGINALPATH));  
    Random random = new Random();  
    StringBuffer buffer = new StringBuffer(  
    "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ");  
    int range = buffer.length();  
    int length = 1;  
    for (int i = 0; i < BIGDATALENGTH; i++) {  
        StringBuffer sb = new StringBuffer();  
        length = random.nextInt(20)+1;  
               //System.out.println("length--->"+length);  
        for (int j = 0; j < length; j++) {  
                       //System.out.println("j--->"+j);  
            sb.append(buffer.charAt(random.nextInt(range)));  
        }  
        System.out.println("sb---->"+sb);  
        writer.write(sb.toString() + "\n");  
    }  
    writer.close();  
}

3.2 对大文件进行切割

Java代码
/** 
 * 将原始数据分成几块 并排序 再保存到临时文件 
 * @throws IOException 
 */  
public static void splitData() throws IOException {  
    @SuppressWarnings("resource")  
    BufferedReader br = new BufferedReader(new FileReader(ORIGINALPATH));  
    tempFiles = new File[BIGDATALENGTH / TEMPFILELENGTH];//将会产生的临时文件列表  
    for (int i = 0; i < tempFiles.length; i++) {  
        tempFiles[i] = new File(TEMPFILEPATH + "TempFile" + i + ".txt");  
        BufferedWriter writer = new BufferedWriter(new FileWriter(tempFiles[i]));  
        HashMap<Integer,String> hashMap = new HashMap<Integer,String>();//未排序  
        //每次读出TEMPFILELENGTH个文件 保存到smallLine中  
        for (int j = 1; j <= TEMPFILELENGTH; j++) {  
            String text = null;  
            if ((text = br.readLine()) != null) {  
                hashMap.put(j, text);  
            }  
        }  
        hashMap = MergeSort.sort(hashMap);  
        for(int k=1; k<=TEMPFILELENGTH; k++){  
            writer.write(String.valueOf(hashMap.get(k))  
                    + System.getProperty("line.separator"));  
//System.getProperty("line.separator")相当于\n  
        }  
        writer.close();  
    }  
}

3.3 对小文件进行递归归并

Java代码
/** 
 * 多路归并排序 
 * @param files 
 * @throws IOException 
 */  
public static void multiWaysMergeSort(String[] files) throws IOException {  
    System.out.println("归并文件-----第 "+mergeSortCount+" 次-----");  
    //当最后只有一个文件的时候 数据已经排序成功 直接复制保存到结果文件  
    if (files.length == 1) {  
        String lastFilePath = LASTFILEPATH + LASTFILENAME;  
        copyFile(files[0], lastFilePath, false);  
        //deleteFile(files[0]);  
        return;  
    }  
    for (int i = 0; i < files.length; i+=2) {  
//开始合并两个相邻的文件 所以一次跳两个  
        if (i == files.length - 1) {  
//这时候已经只剩下最后一个文件了 不需要合并 本趟归并结束  
            renameFile(files[i], i);  
            break;  
        }  
        //将br1 和 br2 写入到Write  
        BufferedReader br1 = new BufferedReader(new FileReader(files[i]));  
        BufferedReader br2 = new BufferedReader(new FileReader(files[i + 1]));  
        BufferedWriter writer = new BufferedWriter(new FileWriter(TEMPFILEPATH + "last_" + mergeSortCount + "_" + i + ".txt"));  
        String s1 = br1.readLine();  
        String s2 = br2.readLine();  
        while (s1 != null || s2 != null) {  
            if (s1 != null && s2 != null) {  
                //都不为空 才有比较的必要  
                int mergeResult = s1.compareTo(s2);  
                if (mergeResult > 0) {//s1在s2后面  
                    writer.write(s2);  
                    writer.write(System.getProperty("line.separator"));  
                    s2 = br2.readLine();  
                }  
                if (mergeResult == 0) {//s1=s2  
                    writer.write(s1);                                   writer.write(System.getProperty("line.separator"));  
                    writer.write(s2);                                   writer.write(System.getProperty("line.separator"));  
                    //System.out.println("write time : " + writeTime++);  
                    s1 = br1.readLine();  
                    s2 = br2.readLine();  
                }  
                if (mergeResult < 0) {//s1在s2前面  
                    writer.write(s1);                                   writer.write(System.getProperty("line.separator"));  
                    s1 = br1.readLine();  
                }  
            }  
            if (s1 == null && s2 != null) {  
                writer.write(s2);  
                writer.write(System.getProperty("line.separator"));  
                s2 = br2.readLine();  
            }  
            if (s2 == null && s1 != null) {  
                writer.write(s1);                 
writer.write(System.getProperty("line.separator"));  
                s1 = br1.readLine();  
            }  
        }  
        br1.close();  
        br2.close();  
//          deleteFile(files[i]);  
//          deleteFile(files[i + 1]);  
        writer.close();  
    }  
    mergeSortCount++;  
    multiWaysMergeSort(getTempFiles("last_" + (mergeSortCount-1) + "_"));  
}