使用fileChannel多线程读取文件数据

使用fileChannel多线程读取文件数据

1.编写文件处理类

import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.LinkedBlockingDeque;

public class MultiThreadReader implements Runnable
{
    private String filePath;

    private long startIndex;

    private long rsize;

    private boolean hasId;

    private int idIndex;

    private String spiltString;

    private Set<String> distinctSet;

    private int bufSize = 1024;

    //换行符
    private int LF = "\n".getBytes()[0];

    //回车
    private int CR = "\r".getBytes()[0];

    public MultiThreadReader(String filePath, long startIndex, long rsize)
    {
        this.filePath = filePath;
        this.startIndex = startIndex > 0 ? startIndex - 1 : startIndex;
        this.rsize = startIndex == 0 ? rsize - 1 : rsize;
    }

    @Override public void run()
    {
        try
        {
            readByLine();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    public void readByLine() throws IOException
    {
        File file = new File(filePath);
        FileChannel fc = new FileInputStream(file).getChannel();
        try
        {
            // 开始按行读取
            ByteBuffer buffer = ByteBuffer.allocate(bufSize);
            byte b;
            List<Byte> bytes = new ArrayList<>();
            String line = null;
            Byte[] by;
            long endIndex = startIndex + rsize;
            boolean isEnd = false;
            boolean ignore = true;
            fc.position(startIndex);
            long endLineIndex = startIndex;
            while (fc.read(buffer) > 0)
            {
                buffer.flip();
                for (int i = 0; i < buffer.limit(); i++)
                {
                    b = buffer.get();
                    if (ignore && b != LF && b != CR && startIndex != 0)
                    {
                        ignore = true;
                        endLineIndex++;
                        continue;
                    }
                    else
                    {
                        ignore = false;
                    }
                    bytes.add(b);
                    if (endLineIndex >= endIndex && (b == LF || b == CR))
                    {
                        // 如果遇到换行
                        getLine(bytes);
                        bytes.clear();
                        isEnd = true;
                        break;
                    }
                    if (b == LF || b == CR)
                    {  // 如果遇到换行
                        getLine(bytes);
                        bytes.clear();
                    }
                    endLineIndex++;
                }
                buffer.clear(); // 清空buffer
                if (isEnd)
                {
                    break;
                }
            }
            //处理最后一行数据
            by = new Byte[bytes.size()];
            line = new String(ArrayUtils.toPrimitive(bytes.toArray(by)), "UTF-8");
            if (!StringUtils.isEmpty(line) && !"\n".equals(line) && !"\r".equals(line))
            {
				System.out.println(line + "\n");
            }
            bytes.clear();
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
        finally
        {
            fc.close();
        }
    }

    /**
     * 获取行数据
     *
     * @param bytes
     * @throws InterruptedException
     */
    public void getLine(List<Byte> bytes) throws InterruptedException
    {
        Byte[] by = new Byte[bytes.size()];
        String line = null;
        try
        {
            line = new String(ArrayUtils.toPrimitive(bytes.toArray(by)), "UTF-8");
        }
        catch (UnsupportedEncodingException e)
        {
            e.printStackTrace();
        }
        if (!StringUtils.isEmpty(line) && !"\n".equals(line) && !"\r".equals(line))
        {
            System.out.println(line);
        }
    }

}

2.调用方法

public class test{

/**
     * 原始文件路径
     */
    private String originFilePath;
    private ExecutorService fixedReadExecutor = Executors.newFixedThreadPool(poolSize);
    /**
     * 处理读取数据的线程数
     */
    private final int poolSize = Runtime.getRuntime().availableProcessors();
    /**
     * 文件头
     */
    private String headString;
	public static void main(String[] args){
		int headLen = (headString + "\n").getBytes().length;
            long threadNum = (byteSize / 1024) == 0 ? 1 : (byteSize / 1024);
            threadNum = threadNum > poolSize ? poolSize : threadNum;
            //平均每个线程读取的字节数
            long subSize = byteSize / threadNum;
            try
            {
                for (int i = 0; i < threadNum; i++)
                {
                    long startIndex = i * subSize + headLen;
                    //处理最后一个线程
                    if (byteSize % threadNum > 0 && i == threadNum - 1)
                    {
                        subSize += byteSize % threadNum + headLen;
                    }
                    fixedReadExecutor.execute(new MultiThreadReader(originFilePath, startIndex, subSize));
                }
            }
            catch (Exception e)
            {
                throw e;
            }
            finally
            {
                fixedReadExecutor.shutdown();
            }
	}
}