我的code

读取一个大的CSV文件，并将数据存入list中

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;

public class ReadLargeFileToList {
    public static void main(String[] args) {
        String filename = "your_large_file.txt";
        String separator = "&*#\\n"; // 换行符字符串
        int bufferSize = 8192; // 8KB 缓冲区大小

        List<String> lines = new ArrayList<>();

        try (FileChannel channel = FileChannel.open(Paths.get(filename), StandardOpenOption.READ)) {
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);

            StringBuilder lineBuilder = new StringBuilder();

            while (channel.read(buffer) > 0) {
                buffer.flip();
                while (buffer.hasRemaining()) {
                    char c = (char) buffer.get();
                    if (separator.indexOf(c) != -1) {
                        lines.add(lineBuilder.toString());
                        lineBuilder.setLength(0); // 清空StringBuilder
                    } else {
                        lineBuilder.append(c);
                    }
                }
                buffer.clear();
            }

            // 处理最后一行，如果文件没有以换行符结尾
            if (lineBuilder.length() > 0) {
                lines.add(lineBuilder.toString());
            }

        } catch (IOException e) {
            e.printStackTrace();
        }

        // 打印读取到的每一行数据
        for (String line : lines) {
            System.out.println(line);
        }
    }
}

读取一个大的list， 把数据写入文件中

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.List;

public class WriteListToFile {
    public static void main(String[] args) {
        List<String> dataList = yourLargeList; // 你的大型列表
        String filename = "output_file.txt";

        Path filePath = Paths.get(filename);

        try {
            // 如果文件已存在，先删除
            if (Files.exists(filePath)) {
                Files.delete(filePath);
            }
            
            // 分块写入数据
            int chunkSize = 1000000; // 每个块的大小
            for (int i = 0; i < dataList.size(); i += chunkSize) {
                int endIndex = Math.min(i + chunkSize, dataList.size());
                List<String> chunk = dataList.subList(i, endIndex);
                Files.write(filePath, chunk, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
            }

            System.out.println("Data has been written to " + filename);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

import java.util.ArrayList;
import java.util.List;

public class SplitListIntoCollections {

    public static void main(String[] args) {
        List<Integer> originalList = new ArrayList<>(); // 你的列表
        for (int i = 1; i <= 100; i++) {
            originalList.add(i); // 添加元素示例
        }

        int numCollections = 10; // 要分成的集合数量
        int elementsPerCollection = originalList.size() / numCollections; // 每个集合的元素数量

        List<List<Integer>> collectionList = new ArrayList<>();
        int startIndex = 0;

        for (int i = 0; i < numCollections - 1; i++) {
            List<Integer> subCollection = originalList.subList(startIndex, startIndex + elementsPerCollection);
            collectionList.add(subCollection); // 将子列表添加到集合列表中
            startIndex += elementsPerCollection;
        }

        // 添加最后一个集合，包含剩余的元素
        List<Integer> lastSubCollection = originalList.subList(startIndex, originalList.size());
        collectionList.add(lastSubCollection);

        // 打印每个小集合
        for (int i = 0; i < collectionList.size(); i++) {
            System.out.println("Collection " + i + ": " + collectionList.get(i));
        }
    }
}

package com.ityj.algorithm.test;

import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

public class ParallelQueueConsumer {

    public static void main(String[] args) {
        // 创建一个队列
        Queue<String> queue = new LinkedList<>();
        // 向队列中添加一些元素
        queue.offer("元素1");
        queue.offer("元素2");
        queue.offer("元素3");
        queue.offer("元素4");
        queue.offer("元素5");

        // 设置并行消费者数量
        int numConsumers = 3;

        // 创建一个固定大小的线程池
        ExecutorService executor = Executors.newFixedThreadPool(numConsumers);

        // 提交消费者任务
        for (int i = 0; i < numConsumers; i++) {
            executor.submit(new QueueConsumer(queue));
        }

        // 关闭线程池
        executor.shutdown();
        try {
            // 等待所有任务执行完毕或者超时
            executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            System.err.println("线程池等待超时！");
            executor.shutdownNow();
        }
    }

    static class QueueConsumer implements Runnable {
        private final Queue<String> queue;

        QueueConsumer(Queue<String> queue) {
            this.queue = queue;
        }

        @Override
        public void run() {
            // 消费队列中的元素
            while (true) {
                String element;
                synchronized (queue) {
                    if (queue.isEmpty()) {
                        // 如果队列为空，则退出循环
                        break;
                    }
                    // 获取并移除队首元素
                    element = queue.poll();
                }
                // 模拟消费过程
                consumeElement(element);
            }
        }

        private void consumeElement(String element) {
            // 模拟消费过程
            System.out.println(Thread.currentThread().getName() + " 消费了：" + element);
            // 可以在这里添加实际的消费逻辑
        }
    }
}