外部排序

当数据量特别大时，大到不可以一次性放入到内存中时，如果我们要对数据排序，那么就要使用外部排序，主要分为两个阶段

1.将数据拆分成我们可以使用内部排序来排序的大小，比如10000，然后将结果写入到临时文件中                                                      2.使用归并排序的思想，将各个临时文件合并成一个

代码实现：

首先创建数据：

public class CreateData {
	public static void main(String[] args) throws IOException {
		DataOutputStream outputStream = FileUtils.getOutputStream("data/largeData.dat");
		
		Random random = new Random();
        //产生的数据量
		for(int i=0;i<100;i++)
			outputStream.writeInt(random.nextInt()%100);
		
		outputStream.close();
		
		DataInputStream inputStream = new DataInputStream(
				new BufferedInputStream(
				new FileInputStream("data/largeData.dat")));
		
		for(int i=0;i<100;i++) {
			if((i+1)%20==0)
				System.out.println();
			System.out.print(inputStream.readInt()+" ");
		}
		inputStream.close();
	}
}

外部排序实现：

//外部排序
public class ExternalSort {
	public static final int MAX_LENGTH = 10;
	
	public static void main(String[] args) throws IOException {
		int[] list = new int[MAX_LENGTH];
		
		int n = readAndSort(list);
		
		mergeResultFile(n);
	}
	
	/**
	 * 首尾合并数据
	 * @param n 文件个数
	 * @throws IOException
	 */
	public static void mergeResultFile(int n) throws IOException {
	    if(n==1) {
	        return;
	    }
	    
	    if(n%2==1) {
    	    for(int i=1;i<n/2+1;i++) {
    	        mergeData("s"+i, "s"+(n-i+1));
    	    }
    	    mergeResultFile(n/2+1);
	    }else {
	        for(int i=1;i<=n/2;i++) {
                mergeData("s"+i, "s"+(n-i+1));
            }
	        mergeResultFile(n/2);
	    }
	}
	
	// 合并两个文件的数据
	public static void mergeData(String s1,String s2) throws IOException {
	    
	    File file = new File("data/temp.dat");
	    if(file.exists()) {
	        file.delete();
	    }
	    DataInputStream input1 = FileUtils.getInputStream("data/"+s1+".dat");
	    DataInputStream input2 = FileUtils.getInputStream("data/"+s2+".dat");
	    DataOutputStream temp = FileUtils.getOutputStream("data/temp.dat");
		
		int a=0,b=0;
		boolean flag1 = false,flag2 = false;
		while(true) {
			try {
			    if(!flag1) {
    				a = input1.readInt();
    				flag1 = true;
			    }
			} catch (EOFException e) {
				break;
			}
			
			try {
			    if(!flag2) {
    				b = input2.readInt();
    				flag2 = true;
			    }
			} catch (EOFException e) {
				break;
			}
			
			// 将小的放到temp中
			if(a>b) {
				temp.writeInt(b);
				flag2 = false;
			}else {
			    temp.writeInt(a);
                flag1 = false;
			}
		}
		
		//文件s2读完,将s1没有读完的复制到temp
		if(flag1) {
			temp.writeInt(a);
			while(true) {
			    try {
	                a = input1.readInt();
	                temp.writeInt(a);
	            } catch (EOFException e) {
	                break;
	            }
			}
		}
		
		//文件s1读完,将s2没有读完的复制到temp
        if(flag2) {
            temp.writeInt(b);
            while(true) {
                try {
                    b = input2.readInt();
                    temp.writeInt(b);
                } catch (EOFException e) {
                    break;
                }
            }
        }
		
		input1.close();
		input2.close();
		temp.close();
		
		copyTempToS1(s1);
	}
	
	/**
	 * 将临时文件的内容写到第一个文件
	 * @param s1
	 * @throws IOException
	 */
	public static void copyTempToS1(String s1) throws IOException{
	    File file = new File("data/"+s1+".dat");
        if(file.exists()) {
            file.delete();
            file.createNewFile();
        }
        DataOutputStream output = FileUtils.getOutputStream("data/"+s1+".dat");
        DataInputStream temp = FileUtils.getInputStream("data/temp.dat");
        int b = 0;
        while(true) {
            try {
                b = temp.readInt();
                output.writeInt(b);
            } catch (EOFException e) {
                break;
            }
        }
        temp.close();
        output.close();
	}
	
	/**
	 * 读取数据，排序后写入临时文件，临时文件命名从s1开始，
	 * @param list
	 * @return 临时文件的个数
	 * @throws IOException
	 */
	public static int readAndSort(int[] list) throws IOException {
		DataInputStream inputStream = 
		        FileUtils.getInputStream("data/largeData.dat");
		
		int index = 0;
		while(true) {
			int len = readData(list, inputStream);
			index++;
			DataOutputStream outputStream = 
			        FileUtils.getOutputStream("data/s"+index+".dat");
                
            MutilThreadSort.sort(list);
            for(int i=0;i<len;i++) {
                outputStream.writeInt(list[i]);
            }
            outputStream.close();
            if(len < MAX_LENGTH) {
                break;
            }
		}
		return index;
	}
	
	// 读取数据，返回长度
	public static int readData(int[] list,DataInputStream inputStream) throws IOException {
		int i = 0,k=0;
		for(i=0;i<MAX_LENGTH;i++) {
			
			try {
				k = inputStream.readInt();
			} catch (EOFException e) {
				break;
			}
			list[i] = k;
		}
		return i;
	}
}

内部排序的实现：

/**
 * forkjoin框架实现多线程排序
 * @author 清明
 *
 */
public class MutilThreadSort {

	public static void sort(int[] list) {
		ForkJoinPool forkJoinPool = new ForkJoinPool();
		forkJoinPool.invoke(new SortTask(list));
	}

	static class SortTask extends RecursiveAction {

		private static final long serialVersionUID = 1L;
		int[] list;
		int max = 1000;

		public SortTask(int[] list) {
			this.list = list;
		}

		@Override
		protected void compute() {
			if (list.length <= max) {
				quickSort(list, 0, list.length - 1);
			} else {
				int mid = list.length / 2;
				int[] first = Arrays.copyOfRange(list, 0, mid);
				int[] second = Arrays.copyOfRange(list, mid, list.length);

				invokeAll(new SortTask(first), new SortTask(second));

				merge(list, first, second);
			}
		}

	}

	// 快速排序
	public static void quickSort(int[] list, int start, int end) {
		if (start >= end)
			return;

		int k = list[start], i = start, j = end;

		while (i < j) {
			while (i < j && list[j] > k) {
				j--;
			}
			if(i<j) {
				list[i] = list[j];
				i++;
			}

			while (i < j && list[i] < k) {
				i++;
			}

			if(i<j) {
				list[j] = list[i];
				j--;
			}
		}
		list[i] = k;
		quickSort(list, start, i-1);
		quickSort(list, i + 1, end);
	}

	// 合并
	public static void merge(int[] list, int[] first, int[] second) {
		int i = 0, j = 0, k = 0;
		while (i < first.length && j < second.length) {
			if (compare(first[i], second[j])) {
				list[k] = first[i];
				i++;
			} else {
				list[k] = second[j];
				j++;
			}
			k++;
		}

		while (i < first.length) {
			list[k++] = first[i++];
		}

		while (j < first.length) {
			list[k++] = first[j++];
		}
		
	}

	// 比大小
	public static boolean compare(int a, int b) {
		return a < b;
	}
}

File工具类：

public class FileUtils {
    
    public static DataInputStream getInputStream(String path) throws FileNotFoundException {
        DataInputStream input = new DataInputStream(
                new BufferedInputStream(
                new FileInputStream(path)));
        return input;
    }
    
    public static DataOutputStream getOutputStream(String path) throws FileNotFoundException {
        DataOutputStream output = new DataOutputStream(
                new BufferedOutputStream(
                new FileOutputStream(path)));
        return output;
    }
}

git地址：https://github.com/201531107001/gqm-practice