新浪面试题-多线程合并文件

今日我收到了一封来自新浪面试的面试邮件，里面就有一道面试题。就是要求多线程合并一个文件，写我整整两天的时间搞定这个题目，给大家分享一下，看代码。

主程序

package com.huang;

import org.apache.log4j.Logger;

import java.util.concurrent.*;

/**
 * 主程序，本程序采用生产者消费者模型，对多个文件进行读写操作
 * Created by root on 16-9-1.
 */
public class Main {
    private static final Logger MAIN_LOGGER=Logger.getLogger(Main.class);
    private static final Logger CON_LOGGER=Logger.getLogger(Consumer.class);
    private static final Logger PRO_LOGGER=Logger.getLogger(Producer.class);
    private static final  ExecutorService POOL;
    //使用阻塞队列实现生产者消费者
    private static final BlockingQueue BLOCKING_QUEUE = new LinkedBlockingQueue();

    static{
        //获取CPU核数
        int cpuNum=Runtime.getRuntime().availableProcessors();
        //因为是IO密集型的程序，如果是CPU密集型程序则N+1
        POOL= Executors.newFixedThreadPool(2*cpuNum+1);
    }

    public static void main(String args[]){
        String in[]={"/tmp/a","/tmp/b","/tmp/c"};
        String out="/tmp/d";
        for(int i=0;i<in.length;i++){
            POOL.execute(new Producer(BLOCKING_QUEUE,in[i],PRO_LOGGER));
        }
        POOL.shutdown();
        new Thread(new Consumer(BLOCKING_QUEUE,CON_LOGGER,out,in.length)).start();
    }

}

生产者

package com.huang;

import org.apache.log4j.Logger;

import java.io.IOException;
import java.util.concurrent.BlockingQueue;


/**
 * 生产者
 * Created by root on 16-8-31.
 */
public class Producer implements Runnable{
    private BlockingQueue<Byte[]> blockingQueue;
    private String filePath;
    private Logger logger;

    public Producer(BlockingQueue blockingQueue,String filePath,Logger logger) {
        this.blockingQueue=blockingQueue;
        this.filePath=filePath;
        this.logger=logger;
    }
    @Override
    public void run() {
        try {
            Byte[] data=FileOperate.toArray(FileOperate.getFileByte(filePath));
            blockingQueue.add(data);
        } catch (IOException e) {
            logger.error(e.getMessage(),e);
        }
    }
}

消费者

package com.huang;

import org.apache.log4j.Logger;

import java.io.IOException;
import java.util.concurrent.BlockingQueue;

/**
 *消费者
 * Created by root on 16-8-31.
 */
public class Consumer implements Runnable{
    private final BlockingQueue<Byte[]> blockingQueue;
    private Logger logger;
    private String outFile;
    private int num;
    public Consumer(BlockingQueue blockingQueue,Logger logger,String outFile,int num){
        this.blockingQueue=blockingQueue;
        this.logger=logger;
        this.outFile=outFile;
        this.num=num;
    }
    @Override
    public void run() {
        while(true){
            try {
                Byte[] bytes=blockingQueue.take();
                FileOperate.writeByte(outFile,bytes);
                if(--num==0)
                    break;
            } catch (InterruptedException e) {
                logger.error(e.getMessage(),e);
            } catch (IOException e) {
                logger.error(e.getMessage(),e);
            }
        }
    }
}

文件操作工具类

package com.huang;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

/**
 * Created by root on 16-8-31.
 */
public class FileOperate {
    /**
     * 使用NIO对文件进行写操作
     * @param filePath
     * @param bytes
     * @throws IOException
     */
    public static void  writeByte(String filePath,Byte[] bytes) throws IOException {
        Lock lock=new ReentrantLock();
        File outFile=new File(filePath);
        if(!outFile.exists())
            outFile.createNewFile();
        try(FileOutputStream fileOutputStream=new FileOutputStream(outFile,true); FileChannel fileChannel=fileOutputStream.getChannel();){
            ByteBuffer bf=ByteBuffer.wrap(getBytes(bytes));
            fileChannel.write(bf);
        }
    }

    /**
     *使用NIO对文件进行读操作
     * @param filePath
     * @return
     * @throws IOException
     */
    public static List<Byte> getFileByte(String filePath) throws IOException {
        File inFile=new File(filePath);
        List<Byte> bytes=null;
        try (FileInputStream fileInputStream= new FileInputStream(filePath); FileChannel fc=fileInputStream.getChannel();){
            bytes=new ArrayList<>();
            ByteBuffer bf=ByteBuffer.allocate(1024);
            while(fc.read(bf)!=-1) {
                bf.flip();
                while(bf.hasRemaining())
                    bytes.add(bf.get());
                bf.clear();
            }
        } catch (FileNotFoundException e) {

            throw new RuntimeException(filePath+"您输入的文件路径有误");
        }

        return bytes;
    }
    public static Byte[] toArray(List<Byte> list){
        Byte[] result=new Byte[list.size()];
        for(int i=0;i<list.size();i++){
            result[i]=list.get(i);
        }
        return result;
    }
    public static byte[] getBytes(Byte[] bytes){
        byte[] result=new byte[bytes.length];
        for(int i=0;i<result.length;i++){
            result[i]=bytes[i];
        }
        return result;
    }
}