多线程读取大文件工具

功能： 多线程读取文件，配置化线程数量、批量处理数量

写在前： 参考借鉴该篇文章，在此基础上做了一些优化JAVA多线程读取同一个文件，加速对文件内容的获取_岁月如歌似梦的博客-CSDN博客

import abc.recommend_plt.model.RecommendModel;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

/**
 *  多线程读取文件
 * @author whvo
 * @date  2023/3/25
 */
@Component
public  class FileReadService {

    /**
     * 多线程读取文件  包含对首行头部的处理
     * @param path         文件地址
     * @param threadCount  线程数量
     * @param batchSize    文件内容批量处理数量
     * @param separator    文件分隔符
     * @param hasTitle     文件是否有标题
     * @param callback     业务处理
     *
     */
    public void readFile(String path,int threadCount,int batchSize, char separator,boolean hasTitle, FileReadService.FileHandleCallback callback){
        if(threadCount<1) {
            threadCount = 1;
        }
        if(StringUtils.isBlank(path)) {
            return ;
        }

        RandomAccessFile randomAccessFile = null;
        try{
            randomAccessFile = new RandomAccessFile(path, "r");
            if(hasTitle) {
                String header = randomAccessFile.readLine();
                if(callback!=null) {
                    callback.setTitle(header);
                }
                randomAccessFile.seek(0);
            }
            callback.setPath(path);
            Long titleLength = getTitleEndIndex(randomAccessFile,separator);
            long fileTotalLength = randomAccessFile.length();
            long gap = fileTotalLength / threadCount;
            long checkIndex = 0;
            long[] beginIndexs = new long[threadCount];
            long[] endIndexs = new long[threadCount];
            for (int n = 0; n < threadCount; n++) {
                beginIndexs[n] = checkIndex;
                if (n + 1 == threadCount) {
                    endIndexs[n] = fileTotalLength;
                    break;
                }
                checkIndex += gap;
                // 将每一页的起始偏移量调整到分割符处
                long gapToEof = getGapToEof(checkIndex, randomAccessFile, separator);
                checkIndex += gapToEof;
                endIndexs[n] = checkIndex;
            }
            // 需要对第一个读取任务增加该偏移量,使其跳过首行title
            if(hasTitle && titleLength + beginIndexs[0] < endIndexs[0]) {
                beginIndexs[0]+=titleLength;
            }
            // 这里可以改为通过容器获取线程池
            ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
            for (int n = 0; n < threadCount; n++) {
                long begin = beginIndexs[n];
                long end = endIndexs[n];
                executorService.execute(() -> {
                    try {
                        readData(path,begin, end, batchSize,null, separator, callback);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                });
            }


        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void readData(String path,long begin, long end,int batchSize, RandomAccessFile randomAccessFile, char separator, FileReadService.FileHandleCallback callback) throws FileNotFoundException, IOException {
        if (randomAccessFile == null) {
            // 初始化随机读取文件对象，模式为只读
            randomAccessFile = new RandomAccessFile(path, "r");
        }
        try{
            List<String>  dataList = new ArrayList<>();
            // 跳转到指定偏移量
            randomAccessFile.seek(begin);
            StringBuilder builder = new StringBuilder();
            while (true) {
                int read = randomAccessFile.read();
                begin++;
                if (separator == read) {
                    dataList.add(builder.toString());
                    builder = new StringBuilder();
                } else {
                    builder.append((char) read);
                }
                if (callback != null && dataList.size() >= batchSize ) {
                    callback.handleBack(dataList);
                    dataList.clear();
                }
                if (begin >= end) {
                    if(CollectionUtils.isNotEmpty(dataList)) {
                        callback.handleBack(dataList);
                    }
                    break;
                }
            }
        }catch (Exception e) {
        }finally {
            if(randomAccessFile!=null) {
                randomAccessFile.close();
            }
        }
    }


    // 以偏移符为基本计算单位
    private long getGapToEof(long beginIndex, RandomAccessFile randomAccessFile, char separator) throws IOException
    {
        randomAccessFile.seek(beginIndex);
        long count = 0;
        while (randomAccessFile.read() != separator) {
            count++;
        }
        count++;
        return count;
    }

    /**
     * 计算title后的第一个偏移点
     * @param randomAccessFile
     * @param separator
     * @return
     * @throws IOException
     */
    private Long getTitleEndIndex(RandomAccessFile randomAccessFile, char separator) throws IOException {
        long index = 0L;
        StringBuilder stringBuilder = new StringBuilder();
        while (true) {
            int read = randomAccessFile.read();
            if (separator == read) {
                return ++index;   //先加再返回
            } else {
                stringBuilder.append((char) read);
                index ++;
            }

        }
    }

    // 对数据做业务处理, 由子类实现
    public static abstract class FileHandleCallback {
        private String title;    // 存放该文件内容的是首行标题
        private String path;     // 存放该文件的路径
        private Map businessMap; // 存放业务参数，可以在初始化时通过有参构造方法传入该参数

        public void setPath(String path) {
            this.path = path;
        }
        public String getPath() {
            return path;
        }
        public void setTitle(String title) {
            this.title = title;
        }
        public String getTitle(){
            return title;
        }
        public void setBusinessMap(Map map) {
            this.businessMap = businessMap;
        }
        public Map getBusinessMap(){
            return this.businessMap;
        }

        public FileHandleCallback(){}

        public FileHandleCallback(Map businessMap) {
            this.businessMap = businessMap;
        }

        abstract void handleBack(List<String> value);
    }
  
    // mock一些测试数据
    private static void mockData() throws  IOException {
        FileOutputStream fileOutputStream = new FileOutputStream("C:\\Users\\10016146\\Desktop\\test.txt");
        for (int n = 0; n < 2000; n++) {
            int count = 3;
            StringBuilder builder = new StringBuilder();
            builder.append("done----").append(n);
            builder.append("\n");
            fileOutputStream.write(builder.toString().getBytes());
        }
        fileOutputStream.close();
    }
   
        public static void main(String[] args) throws IOException {
//        mockData();
        // 这个map里面放一些业务用的参数，可以透传到最后的handleBack中
        Map<String,String> business = new HashMap<>();
        business.put("xxx1","xxx");
        business.put("xxx","xx");
        Boolean hasTitle = true;
        FileReadService helper = new FileReadService();
        helper.readFile("文件地址", 1, 50,'\n',true, new FileReadService.FileHandleCallback(business) {
         // 重写handleBack方法，实现业务，businessHandle方法就是业务代码，已省略
            @Override
            void handleBack(List<String> value) {
//                System.out.println(this.getPath() +">>"+ this.getTitle() +">>"+value);
                businessHandle(this.getBusinessMap(),this.getPath(),this.getTitle(),value);
            }
        });
   }
}