外部排序优化

优化外部排序
主要步骤重新拆分如下
第一阶段
1.将未排序的大文件切分(spilterReceiver)
2.一个线程池专门接收byte[]，并将其转化为long[] (prepareReceiver)
3.一个线程池专门排序long[] (sortReceiver)
4.一个线程池专门将已经有序的long[]写入文件 (writerReceiver)
这个过程直接写long类型的数字到文件，因为写入文本占用IO资源比直接写入long型数字要多。

第二阶段
1.一个线程用于归并排序,将结果写入缓冲区 (mergerReceiver)
2.如果缓冲区快满了，就将其交给一个线程池用于写入最终的文件。 (mergerReceiver)

优化的重点是平衡CPU，内存和硬盘的资源使用。

每个电脑的配置不同，都需要进行不同的调整，才能达到最优化的效果。
单位的电脑是i5的CPU，8G内存,7200转硬盘。
我使用的虚拟机分配了2个核心,4G内存。
跑这个程序大致需要200s左右。
因为第二阶段的第一个步骤是非常消耗CPU资源的，单位电脑CPU主频很高，所以不会成为瓶颈。

而家里的电脑，应该配置更接近吴老师做实验的配置。主频比较低,32位系统,4G内存。
这个配置明显CPU成为瓶颈
在第一个阶段，切分出来的文件，由于CPU不能及时排序并输出，经常导致OOM
只能在切分一个文件出来之后，让切分线程sleep一段时间，大量测试得到的结果是这个线程至少sleep2500毫秒才能不导致OOM。
也就是说,CPU运算的速度跟不上IO的速度。后来想到一个方法，加大分片。原来4G文件使用15个分片，现在使用30个分片。数据快入快出,这样可以避免OOM。至于这个分片的数字，恐怕每个机器都需要实际的测试。
另外在第二阶段，也出现了CPU拖累IO,导致性能下降。
因为归并的时候，需要大量运算各个分片中最小的数据，经常看到计算50M数据,CPU需要2s，而写入一般1s就可以完成。
而这个过程是单线程的，CPU的主频直接决定了外部排序的总体时间。

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class Controller {
    public static void main(String[] args) throws IOException {
        new Controller().action(new File("/home/lihuilin/桌面/t.txt"), 30, "/home/lihuilin/桌面/");

    }

    public void action(File file, int pieces, String outDir) throws IOException {
        Invoker invoker = new Invoker(pieces);
        List<Command> commandList = blocking(file, pieces, outDir);
        for (Command command : commandList) {
            command.setInvoker(invoker);
            invoker.executeCommand(command);
        }
    }

    private List<Command> blocking(File file, int pieces, String outDir) throws IOException {
        List<Command> result = new ArrayList<Command>();
        List<Long> list = new ArrayList<Long>();
        list.add(-1L);
        long length = file.length();
        long step = length / pieces;
        long index = 0;
        for (int i = 0; i < pieces; i++) {
            BufferedInputStream in = new BufferedInputStream(new FileInputStream(file));
            if (index + step < length) {
                index = index + step;
                in.skip(index);
                while (in.read() != 10) {
                    index = index + 1;
                }
                list.add(index);
                index++;
            }
            in.close();
        }
        list.add(length - 1);
        for (int i = 0; i < list.size() - 1; i++) {
            long skipSize = list.get(i) + 1;
            long l = list.get(i + 1) - list.get(i);
            result.add(new SpilterCommand(file, skipSize, l, outDir));
        }
        return result;
    }

}

interface Command extends Comparable<Command> {
    public void setInvoker(Invoker invoker);

    public void setReceiver(Receiver receiver);

    public void execute() throws IOException;

    public long getSkipSize();

    public int getLength();

    public String getOutDir();

    public File getSourceFile();

    public Invoker getInvoker();

    public void clear();

    public File getOutFile();

    public File getTargetFile();
}

class Invoker {
    private Receiver sortReceiver;
    private Receiver spilterReceiver;
    private Receiver writerReceiver;
    private Receiver prepareReceiver;
    private Receiver mergerReceiver;

    public Invoker(int pieces) {
        writerReceiver = new WriterReceiver(pieces);
        spilterReceiver = new SpilterReceiver();
        sortReceiver = new SorterReceiver();
        prepareReceiver = new PrepareReceiver();
        mergerReceiver = new MergerReceiver();
    }

    public void executeCommand(Command command) {
        try {
            if (command instanceof SpilterCommand) {
                spilterReceiver.action(command);
            } else if (command instanceof PrepareCommand) {
                prepareReceiver.action(command);
            } else if (command instanceof SortCommand) {
                sortReceiver.action(command);
            } else if (command instanceof WriterCommand) {
                writerReceiver.action(command);
            } else if (command instanceof MergeCommand) {
                mergerReceiver.action(command);
            } else if (command instanceof KillCommand) {
                writerReceiver.shutdown();
                spilterReceiver.shutdown();
                sortReceiver.shutdown();
                prepareReceiver.shutdown();
                mergerReceiver.shutdown();
            }
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }
}

class SpilterCommand extends AbstractCommand {

    public SpilterCommand(File sourceFile, long skipSize, long length, String outDir) {
        super(sourceFile, skipSize, length, outDir);
    }

    @Override
    public void clear() {

    }

}

abstract class AbstractCommand implements Command {
    private Invoker invoker = null;
    private Receiver receiver = null;
    private long skipSize = 0L;
    private int length = 0;
    private String outDir = null;
    private File sourceFile;
    private File outFile = null;
    public static final long START = System.currentTimeMillis();
    private static int FLAG = 0;
    private File targetFile = null;

    private synchronized int getFlag() {
        return FLAG++;
    }

    public AbstractCommand(Command command) {
        this(command.getSourceFile(), command.getSkipSize(), command.getLength(), command.getOutDir());
        this.invoker = command.getInvoker();
        if (command.getOutFile() != null) {
            this.outFile = command.getOutFile();
        } else {
            this.outFile = new File(this.outDir + getFlag() + ".temp");
        }
    }

    public AbstractCommand(File sourceFile, long skipSize, long length, String outDir) {
        if (length > Integer.MAX_VALUE) {
            throw new RuntimeException("长度溢出");
        }
        this.skipSize = skipSize;
        this.length = (int) length;
        this.outDir = outDir;
        this.sourceFile = sourceFile;
        this.targetFile = new File(outDir + "result.temp");
    }

    public void setInvoker(Invoker invoker) {
        this.invoker = invoker;
    }

    public void setReceiver(Receiver receiver) {
        this.receiver = receiver;
    }

    public long getSkipSize() {
        return skipSize;
    }

    public int getLength() {
        return length;
    }

    public String getOutDir() {
        return outDir;
    }

    public File getSourceFile() {
        return sourceFile;
    }

    @Override
    public String toString() {
        return "Command [skipSize=" + skipSize + ", length=" + length + "]";
    }

    public void execute() throws IOException {
        this.receiver.action(this);
    }

    public Invoker getInvoker() {
        return invoker;
    }

    public File getOutFile() {
        return this.outFile;
    }

    public File getTargetFile() {
        return this.targetFile;
    }

    @Override
    public int compareTo(Command o) {
        if (this.skipSize < o.getSkipSize()) {
            return -1;
        }
        return 1;
    }
}

interface Receiver {
    public void action(Command command) throws IOException;

    public void shutdown();
}

class PrepareReceiver implements Receiver {
    private ExecutorService prepareThreadPool;

    public PrepareReceiver() {
        prepareThreadPool = Executors.newFixedThreadPool(4);
    }

    @Override
    public void action(final Command command) throws IOException {
        prepareThreadPool.submit(new Runnable() {

            @Override
            public void run() {
                System.out.println("\t整理数据:" + command);
                long start = System.currentTimeMillis();

                byte[] data = ((PrepareCommand) command).getData();
                long[] result = new long[getObjectSize(data)];
                int resultIndex = 0;
                int index = 0;
                int first = 0;
                while (index < data.length) {
                    if (data[index] == 10) {
                        byte[] tmpData = Arrays.copyOfRange(data, first, index);
                        String str = new String(tmpData);
                        result[resultIndex] = Long.valueOf(str);
                        resultIndex++;
                        first = index + 1;
                    }

                    index++;
                }
                command.getInvoker().executeCommand(new SortCommand(command, result));

                result = null;
                data = null;
                command.clear();

                long end = System.currentTimeMillis();
                System.out.println("\t结束整理数据:" + command + ",用时:" + (end - start) / 1000);
            }

        });

    }

    private int getObjectSize(byte[] data) {
        int size = 0;
        for (byte b : data) {
            if (b == 10) {
                size++;
            }
        }
        return size;
    }

    @Override
    public void shutdown() {
        this.prepareThreadPool.shutdown();
    }

}

class PrepareCommand extends AbstractCommand {

    private byte[] data;

    public PrepareCommand(Command command, byte[] data) {
        super(command);
        this.data = data;
    }

    public byte[] getData() {
        return this.data;
    }

    @Override
    public void clear() {
        this.data = null;
    }
}

class SpilterReceiver implements Receiver {
    @Override
    public void action(final Command command) throws IOException {
        try {
            System.out.println("开始读入:" + command);
            long start = System.currentTimeMillis();
            FileChannel in = new RandomAccessFile(command.getSourceFile(), "r").getChannel();
            MappedByteBuffer inBuffer = in.map(MapMode.READ_ONLY, command.getSkipSize(), command.getLength());
            byte[] data = new byte[inBuffer.limit()];
            inBuffer.get(data);
            command.getInvoker().executeCommand(new PrepareCommand(command, data));

            data = null;
            in.close();
            command.clear();

            long end = System.currentTimeMillis();
            System.out.println("结束内存读入:" + (end - start) / 1000 + "s");

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Override
    public void shutdown() {
    }
}

class MergeCommand extends AbstractCommand {
    private List<Command> commandList = null;

    public MergeCommand(List<Command> commandList) {
        super(commandList.get(0));
        this.commandList = commandList;
    }

    @Override
    public void clear() {

    }

    public List<Command> getCommandList() {
        return this.commandList;
    }
}

class KillCommand extends AbstractCommand {

    public KillCommand(Command command) {
        super(command);
    }

    @Override
    public void clear() {

    }

}

class MergerReceiver implements Receiver {
    private ExecutorService mergerThreadPool = Executors.newFixedThreadPool(4);
    private List<Worker> workerList = new ArrayList<Worker>();
    private ByteBuffer bb = null;

    @Override
    public void action(final Command command) throws IOException {
        bb = ByteBuffer.allocate(50 * 1024 * 1024);
        List<Command> list = ((MergeCommand) command).getCommandList();
        for (Command historyCommand : list) {
            Worker worker = new Worker(historyCommand.getOutFile(), workerList);
            workerList.add(worker);
        }

        System.out.println("读取队列,写入目标文件");
        long start = System.currentTimeMillis();
        RandomAccessFile targetFile = new RandomAccessFile(command.getTargetFile(), "rw");
        FileChannel channel = targetFile.getChannel();
        MappedByteBuffer out = null;
        long index = 0L;
        while (workerList.size() != 0) {
            Worker worker = Collections.min(workerList);
            Long data = worker.poll();
            if (data == null) {
                workerList.remove(worker);
            } else {

                bb.put((data + "\n").getBytes());

                if (bb.position() > 49 * 1024 * 1024) {
                    long end = System.currentTimeMillis();
                    System.out.println("归并用时:" + (end - start)/1000 + "s");
                    start = System.currentTimeMillis();
                    bb.flip();
                    long temp = bb.limit();
                    MutiWriter writer = new MutiWriter(channel, bb, index);
                    mergerThreadPool.submit(writer);
                    index = index + temp;
                    bb = ByteBuffer.allocate(50 * 1024 * 1024);
                }

            }
        }

        bb.flip();

        out = channel.map(MapMode.READ_WRITE, index, bb.limit());
        int i = bb.limit();
        while (i > .0) {
            out.put(bb.get());
            i--;
        }
        out.force();
        channel.close();
        targetFile.close();

        KillCommand killCommand = new KillCommand(command);
        command.getInvoker().executeCommand(killCommand);

        long end = System.currentTimeMillis();
        System.out.println("外部排序总用时:" + (end - AbstractCommand.START) / 1000);

    }

    private class MutiWriter implements Runnable {
        private ByteBuffer buffer;
        private MappedByteBuffer out = null;

        public MutiWriter(FileChannel channel, ByteBuffer buffer, long index) {
            this.buffer = buffer;
            try {
                out = channel.map(MapMode.READ_WRITE, index, buffer.limit());
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        @Override
        public void run() {
            long a1 = System.currentTimeMillis();
            int i = buffer.limit();
            while (i > 0) {
                out.put(buffer.get());
                i--;
            }
            out.force();
            out = null;
            long a2 = System.currentTimeMillis();
            System.out.println("内存映射写入:" + (a2 - a1) / 1000 + "s");
        }

    }

    @Override
    public void shutdown() {
        mergerThreadPool.shutdown();
    }

    private class Worker implements Comparable<Worker> {
        private long data;
        private MappedByteBuffer buffer = null;
        private List<Worker> workerList = null;
        private boolean eof = false;

        Worker(File file, List<Worker> workerList) {
            try {
                RandomAccessFile rFile = new RandomAccessFile(file, "r");
                FileChannel channel = rFile.getChannel();
                buffer = channel.map(MapMode.READ_ONLY, 0, channel.size());

                this.workerList = workerList;
                data = buffer.getLong();

                rFile.close();
                channel.close();

            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }

        public long peek() {
            return data;
        }

        public Long poll() {
            long result = data;
            if (buffer.position() != buffer.limit()) {
                data = buffer.getLong();
            } else {
                if (eof == false) {
                    eof = true;
                } else {
                    return null;
                }
            }

            return result;
        }

        @Override
        public int compareTo(Worker o) {
            if (this.peek() > o.peek()) {
                return 1;
            } else if (this.peek() < o.peek()) {
                return -1;
            } else {
                return 0;
            }
        }
    }

}

class WriterReceiver implements Receiver {
    private ExecutorService writeThreadPool = null;;
    private CyclicBarrier barrier = null;
    private List<Command> commandList = new ArrayList<Command>();

    public WriterReceiver(int pieces) {
        writeThreadPool = Executors.newFixedThreadPool(pieces + 2);
        barrier = new CyclicBarrier(pieces, new Runnable() {

            @Override
            public void run() {
                long end = System.currentTimeMillis();
                System.out.println("合并之前总用时:" + (end - AbstractCommand.START) / 1000);

                MergeCommand command = new MergeCommand(WriterReceiver.this.commandList);
                command.getInvoker().executeCommand(command);

            }
        });

    }

    @Override
    public void action(final Command command) {
        writeThreadPool.submit(new Runnable() {

            @Override
            public void run() {
                long[] data = ((WriterCommand) command).getData();

                try {
                    System.out.println("\t\t\t开始写入:" + command);
                    long start = System.currentTimeMillis();
                    File outfile = command.getOutFile();
                    FileChannel channel = new RandomAccessFile(outfile, "rw").getChannel();
                    MappedByteBuffer buffer = channel.map(MapMode.READ_WRITE, 0, data.length * 8);
                    for (int i = 0; i < data.length; i++) {
                        buffer.putLong(data[i]);
                    }
                    buffer.force();
                    WriterReceiver.this.commandList.add(command);
                    long end = System.currentTimeMillis();
                    System.out.println("\t\t\t结束写入:" + command + ",用时:" + (end - start) / 1000);

                    command.clear();
                    data = null;
                    channel.close();
                    buffer = null;

                    barrier.await();
                } catch (IOException ex) {
                    ex.printStackTrace();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (BrokenBarrierException e) {
                    e.printStackTrace();
                }
            }
        });
    }

    @Override
    public void shutdown() {
        writeThreadPool.shutdown();
    }
}

class SortCommand extends AbstractCommand {
    private long[] data = null;

    public SortCommand(Command command, long[] data) {
        super(command);
        this.data = data;
    }

    public long[] getData() {
        return this.data;
    }

    @Override
    public void clear() {
        this.data = null;
    }

}

class SorterReceiver implements Receiver {
    private ExecutorService sortThreadPool = Executors.newFixedThreadPool(4);

    public void action(final Command command) {
        sortThreadPool.submit(new Runnable() {

            @Override
            public void run() {
                System.out.println("\t\t开始排序:" + command);
                long start = System.currentTimeMillis();

                long[] data = ((SortCommand) command).getData();
                Arrays.sort(data);

                command.getInvoker().executeCommand(new WriterCommand(command, data));

                data = null;
                command.clear();

                long end = System.currentTimeMillis();
                System.out.println("\t\t结束排序:" + command + ",用时:" + (end - start) / 1000);

            }

        });

    }

    @Override
    public void shutdown() {
        sortThreadPool.shutdown();
    }
}

class WriterCommand extends AbstractCommand {
    private long[] data;

    public WriterCommand(Command command, long[] data) {
        super(command);
        this.data = data;
    }

    public long[] getData() {
        return this.data;
    }

    @Override
    public void clear() {
        this.data = null;
    }

}

最近真的很忙,我这一个单线程的CPU居然都跑出了双核的感觉。
我感觉优化就是找到慢的地方，看看是不是有无效的操作,如果都是有效运算,就只能想办法使用多线程，使压力分散在多个核心上。
之前吴老师说他用的是直接缓冲区+通道 IO的方式。自己写程序将long直接转化为byte。
这个优化的方式我试了一下，不能应用在我的场景。
他的本意是减少GC，但是在我的场景，这种方式更加剧了CPU的争用。
优化的本质应该是平衡资源

程序日志,相比最初的版本，性能提升了50s
因为将切分文件，byte数组转为long数组拆分为了两个操作，并且使用了多线程处理。

开始读入:Command [skipSize=0, length=135863265] 结束内存读入:3s 整理数据:Command [skipSize=0, length=135863265] 开始读入:Command [skipSize=135863265, length=135863254] 结束整理数据:Command [skipSize=0, length=135863265],用时:4 开始排序:Command [skipSize=0, length=135863265] 结束内存读入:4s 开始读入:Command [skipSize=271726519, length=135863267] 整理数据:Command [skipSize=135863265, length=135863254] 结束排序:Command [skipSize=0, length=135863265],用时:2 开始写入:Command [skipSize=0, length=135863265] 结束内存读入:3s 开始读入:Command [skipSize=407589786, length=135863268] 整理数据:Command [skipSize=271726519, length=135863267] 结束写入:Command [skipSize=0, length=135863265],用时:2 结束整理数据:Command [skipSize=135863265, length=135863254],用时:5 开始排序:Command [skipSize=135863265, length=135863254] 结束排序:Command [skipSize=135863265, length=135863254],用时:1 开始写入:Command [skipSize=135863265, length=135863254] 结束整理数据:Command [skipSize=271726519, length=135863267],用时:4 开始排序:Command [skipSize=271726519, length=135863267] 结束排序:Command [skipSize=271726519, length=135863267],用时:1 开始写入:Command [skipSize=271726519, length=135863267] 结束写入:Command [skipSize=135863265, length=135863254],用时:6 结束写入:Command [skipSize=271726519, length=135863267],用时:4 结束内存读入:11s 整理数据:Command [skipSize=407589786, length=135863268] 开始读入:Command [skipSize=543453054, length=135863264] 结束整理数据:Command [skipSize=407589786, length=135863268],用时:3 开始排序:Command [skipSize=407589786, length=135863268] 结束内存读入:4s 开始读入:Command [skipSize=679316318, length=135863268] 整理数据:Command [skipSize=543453054, length=135863264] 结束排序:Command [skipSize=407589786, length=135863268],用时:1 开始写入:Command [skipSize=407589786, length=135863268] 结束写入:Command [skipSize=407589786, length=135863268],用时:1 结束整理数据:Command [skipSize=543453054, length=135863264],用时:3 开始排序:Command [skipSize=543453054, length=135863264] 结束排序:Command [skipSize=543453054, length=135863264],用时:1 开始写入:Command [skipSize=543453054, length=135863264] 结束写入:Command [skipSize=543453054, length=135863264],用时:1 结束内存读入:7s 整理数据:Command [skipSize=679316318, length=135863268] 开始读入:Command [skipSize=815179586, length=135863264] 结束内存读入:3s 开始读入:Command [skipSize=951042850, length=135863270] 整理数据:Command [skipSize=815179586, length=135863264] 开始排序:Command [skipSize=679316318, length=135863268] 结束整理数据:Command [skipSize=679316318, length=135863268],用时:3 结束排序:Command [skipSize=679316318, length=135863268],用时:2 开始写入:Command [skipSize=679316318, length=135863268] 结束整理数据:Command [skipSize=815179586, length=135863264],用时:4 开始排序:Command [skipSize=815179586, length=135863264] 结束写入:Command [skipSize=679316318, length=135863268],用时:2 结束内存读入:5s 整理数据:Command [skipSize=951042850, length=135863270] 开始读入:Command [skipSize=1086906120, length=135863262] 结束排序:Command [skipSize=815179586, length=135863264],用时:1 开始写入:Command [skipSize=815179586, length=135863264] 结束写入:Command [skipSize=815179586, length=135863264],用时:1 整理数据:Command [skipSize=1086906120, length=135863262] 结束内存读入:3s 开始读入:Command [skipSize=1222769382, length=135863265] 开始排序:Command [skipSize=951042850, length=135863270] 结束整理数据:Command [skipSize=951042850, length=135863270],用时:4 结束内存读入:3s 开始读入:Command [skipSize=1358632647, length=135863256] 整理数据:Command [skipSize=1222769382, length=135863265] 结束排序:Command [skipSize=951042850, length=135863270],用时:2 开始写入:Command [skipSize=951042850, length=135863270] 结束写入:Command [skipSize=951042850, length=135863270],用时:1 结束整理数据:Command [skipSize=1086906120, length=135863262],用时:5 开始排序:Command [skipSize=1086906120, length=135863262] 结束排序:Command [skipSize=1086906120, length=135863262],用时:1 开始写入:Command [skipSize=1086906120, length=135863262] 结束内存读入:4s 整理数据:Command [skipSize=1358632647, length=135863256] 开始读入:Command [skipSize=1494495903, length=135863268] 结束写入:Command [skipSize=1086906120, length=135863262],用时:1 开始排序:Command [skipSize=1222769382, length=135863265] 结束整理数据:Command [skipSize=1222769382, length=135863265],用时:6 结束排序:Command [skipSize=1222769382, length=135863265],用时:1 开始写入:Command [skipSize=1222769382, length=135863265] 整理数据:Command [skipSize=1494495903, length=135863268] 结束内存读入:4s 开始读入:Command [skipSize=1630359171, length=135863255] 开始排序:Command [skipSize=1358632647, length=135863256] 结束整理数据:Command [skipSize=1358632647, length=135863256],用时:5 结束写入:Command [skipSize=1222769382, length=135863265],用时:1 结束排序:Command [skipSize=1358632647, length=135863256],用时:1 开始写入:Command [skipSize=1358632647, length=135863256] 结束整理数据:Command [skipSize=1494495903, length=135863268],用时:3 结束写入:Command [skipSize=1358632647, length=135863256],用时:2 开始排序:Command [skipSize=1494495903, length=135863268] 结束内存读入:4s 整理数据:Command [skipSize=1630359171, length=135863255] 开始读入:Command [skipSize=1766222426, length=135863271] 结束排序:Command [skipSize=1494495903, length=135863268],用时:1 开始写入:Command [skipSize=1494495903, length=135863268] 结束写入:Command [skipSize=1494495903, length=135863268],用时:1 结束整理数据:Command [skipSize=1630359171, length=135863255],用时:3 开始排序:Command [skipSize=1630359171, length=135863255] 整理数据:Command [skipSize=1766222426, length=135863271] 结束内存读入:4s 开始读入:Command [skipSize=1902085697, length=135863258] 结束排序:Command [skipSize=1630359171, length=135863255],用时:2 开始写入:Command [skipSize=1630359171, length=135863255] 结束写入:Command [skipSize=1630359171, length=135863255],用时:1 结束整理数据:Command [skipSize=1766222426, length=135863271],用时:3 开始排序:Command [skipSize=1766222426, length=135863271] 整理数据:Command [skipSize=1902085697, length=135863258] 结束内存读入:4s 开始读入:Command [skipSize=2037948955, length=135863254] 结束排序:Command [skipSize=1766222426, length=135863271],用时:1 开始写入:Command [skipSize=1766222426, length=135863271] 结束写入:Command [skipSize=1766222426, length=135863271],用时:1 结束整理数据:Command [skipSize=1902085697, length=135863258],用时:3 开始排序:Command [skipSize=1902085697, length=135863258] 整理数据:Command [skipSize=2037948955, length=135863254] 结束内存读入:4s 开始读入:Command [skipSize=2173812209, length=135863259] 结束排序:Command [skipSize=1902085697, length=135863258],用时:2 开始写入:Command [skipSize=1902085697, length=135863258] 结束写入:Command [skipSize=1902085697, length=135863258],用时:1 结束整理数据:Command [skipSize=2037948955, length=135863254],用时:3 开始排序:Command [skipSize=2037948955, length=135863254] 结束内存读入:4s 整理数据:Command [skipSize=2173812209, length=135863259] 开始读入:Command [skipSize=2309675468, length=135863260] 结束排序:Command [skipSize=2037948955, length=135863254],用时:2 开始写入:Command [skipSize=2037948955, length=135863254] 结束整理数据:Command [skipSize=2173812209, length=135863259],用时:3 开始排序:Command [skipSize=2173812209, length=135863259] 结束内存读入:4s 整理数据:Command [skipSize=2309675468, length=135863260] 开始读入:Command [skipSize=2445538728, length=135863266] 结束写入:Command [skipSize=2037948955, length=135863254],用时:2 结束排序:Command [skipSize=2173812209, length=135863259],用时:2 开始写入:Command [skipSize=2173812209, length=135863259] 结束写入:Command [skipSize=2173812209, length=135863259],用时:1 结束整理数据:Command [skipSize=2309675468, length=135863260],用时:3 开始排序:Command [skipSize=2309675468, length=135863260] 整理数据:Command [skipSize=2445538728, length=135863266] 结束内存读入:4s 开始读入:Command [skipSize=2581401994, length=135863268] 结束排序:Command [skipSize=2309675468, length=135863260],用时:2 开始写入:Command [skipSize=2309675468, length=135863260] 结束写入:Command [skipSize=2309675468, length=135863260],用时:1 结束整理数据:Command [skipSize=2445538728, length=135863266],用时:4 开始排序:Command [skipSize=2445538728, length=135863266] 结束内存读入:4s 整理数据:Command [skipSize=2581401994, length=135863268] 开始读入:Command [skipSize=2717265262, length=135863264] 结束排序:Command [skipSize=2445538728, length=135863266],用时:2 开始写入:Command [skipSize=2445538728, length=135863266] 结束写入:Command [skipSize=2445538728, length=135863266],用时:1 结束整理数据:Command [skipSize=2581401994, length=135863268],用时:3 开始排序:Command [skipSize=2581401994, length=135863268] 整理数据:Command [skipSize=2717265262, length=135863264] 结束内存读入:4s 开始读入:Command [skipSize=2853128526, length=135863258] 结束排序:Command [skipSize=2581401994, length=135863268],用时:2 开始写入:Command [skipSize=2581401994, length=135863268] 结束写入:Command [skipSize=2581401994, length=135863268],用时:1 结束整理数据:Command [skipSize=2717265262, length=135863264],用时:3 开始排序:Command [skipSize=2717265262, length=135863264] 整理数据:Command [skipSize=2853128526, length=135863258] 结束内存读入:4s 开始读入:Command [skipSize=2988991784, length=135863254] 结束排序:Command [skipSize=2717265262, length=135863264],用时:2 开始写入:Command [skipSize=2717265262, length=135863264] 结束写入:Command [skipSize=2717265262, length=135863264],用时:1 结束整理数据:Command [skipSize=2853128526, length=135863258],用时:3 开始排序:Command [skipSize=2853128526, length=135863258] 结束内存读入:4s 整理数据:Command [skipSize=2988991784, length=135863254] 开始读入:Command [skipSize=3124855038, length=135863260] 结束排序:Command [skipSize=2853128526, length=135863258],用时:2 开始写入:Command [skipSize=2853128526, length=135863258] 结束写入:Command [skipSize=2853128526, length=135863258],用时:1 结束整理数据:Command [skipSize=2988991784, length=135863254],用时:3 开始排序:Command [skipSize=2988991784, length=135863254] 整理数据:Command [skipSize=3124855038, length=135863260] 结束内存读入:4s 开始读入:Command [skipSize=3260718298, length=135863254] 结束排序:Command [skipSize=2988991784, length=135863254],用时:2 开始写入:Command [skipSize=2988991784, length=135863254] 结束写入:Command [skipSize=2988991784, length=135863254],用时:2 结束整理数据:Command [skipSize=3124855038, length=135863260],用时:4 开始排序:Command [skipSize=3124855038, length=135863260] 整理数据:Command [skipSize=3260718298, length=135863254] 结束内存读入:4s 开始读入:Command [skipSize=3396581552, length=135863262] 结束排序:Command [skipSize=3124855038, length=135863260],用时:1 开始写入:Command [skipSize=3124855038, length=135863260] 结束写入:Command [skipSize=3124855038, length=135863260],用时:2 结束整理数据:Command [skipSize=3260718298, length=135863254],用时:3 开始排序:Command [skipSize=3260718298, length=135863254] 结束内存读入:4s 整理数据:Command [skipSize=3396581552, length=135863262] 开始读入:Command [skipSize=3532444814, length=135863265] 结束排序:Command [skipSize=3260718298, length=135863254],用时:1 开始写入:Command [skipSize=3260718298, length=135863254] 结束写入:Command [skipSize=3260718298, length=135863254],用时:2 结束整理数据:Command [skipSize=3396581552, length=135863262],用时:3 开始排序:Command [skipSize=3396581552, length=135863262] 结束内存读入:4s 整理数据:Command [skipSize=3532444814, length=135863265] 开始读入:Command [skipSize=3668308079, length=135863273] 结束排序:Command [skipSize=3396581552, length=135863262],用时:2 开始写入:Command [skipSize=3396581552, length=135863262] 结束写入:Command [skipSize=3396581552, length=135863262],用时:2 结束整理数据:Command [skipSize=3532444814, length=135863265],用时:3 开始排序:Command [skipSize=3532444814, length=135863265] 结束内存读入:4s 整理数据:Command [skipSize=3668308079, length=135863273] 开始读入:Command [skipSize=3804171352, length=135863261] 结束排序:Command [skipSize=3532444814, length=135863265],用时:2 开始写入:Command [skipSize=3532444814, length=135863265] 结束写入:Command [skipSize=3532444814, length=135863265],用时:1 结束整理数据:Command [skipSize=3668308079, length=135863273],用时:3 开始排序:Command [skipSize=3668308079, length=135863273] 整理数据:Command [skipSize=3804171352, length=135863261] 结束内存读入:4s 开始读入:Command [skipSize=3940034613, length=135862991] 结束排序:Command [skipSize=3668308079, length=135863273],用时:2 开始写入:Command [skipSize=3668308079, length=135863273] 结束写入:Command [skipSize=3668308079, length=135863273],用时:2 结束整理数据:Command [skipSize=3804171352, length=135863261],用时:3 开始排序:Command [skipSize=3804171352, length=135863261] 结束内存读入:4s 整理数据:Command [skipSize=3940034613, length=135862991] 结束排序:Command [skipSize=3804171352, length=135863261],用时:1 开始写入:Command [skipSize=3804171352, length=135863261] 结束写入:Command [skipSize=3804171352, length=135863261],用时:1 结束整理数据:Command [skipSize=3940034613, length=135862991],用时:3 开始排序:Command [skipSize=3940034613, length=135862991] 结束排序:Command [skipSize=3940034613, length=135862991],用时:1 开始写入:Command [skipSize=3940034613, length=135862991] 结束写入:Command [skipSize=3940034613, length=135862991],用时:1 合并之前总用时:143 读取队列,写入目标文件 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:1s 归并用时:2s 归并用时:1s 内存映射写入:3s 归并用时:2s 归并用时:2s 内存映射写入:5s 内存映射写入:3s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:1s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:4s 内存映射写入:2s 归并用时:3s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 归并用时:1s 归并用时:2s 归并用时:4s 内存映射写入:6s 内存映射写入:8s 内存映射写入:4s 归并用时:2s 归并用时:2s 内存映射写入:3s 内存映射写入:5s 内存映射写入:2s 归并用时:3s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:1s 归并用时:3s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 归并用时:2s 内存映射写入:2s 内存映射写入:2s 归并用时:2s 内存映射写入:1s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:2s 归并用时:2s 内存映射写入:1s 外部排序总用时:351

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/29254281/viewspace-1172752/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/29254281/viewspace-1172752/