模拟kafka测试零拷贝和磁盘顺序写的速度

接收端

使用nio + io多路复用 + 零拷贝 + 磁盘顺序写

package com.chun.multiplexing;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.*;
import java.util.Iterator;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;

/**
 * @author chun
 * @date 2023/4/7 10:14
 */
public class FileReceiver {
    private static final int BUFFER_SIZE = 1024 * 1024 * 64;

    public static void main(String[] args) {
        ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(20);
        try {
            Selector selector = Selector.open();

            ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
            serverSocketChannel.bind(new InetSocketAddress(9999));
            serverSocketChannel.configureBlocking(false);
            serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);

            while (true) {
                int nkeys = selector.select();
                if (nkeys > 0) {
                    Iterator<SelectionKey> selectionKeys = selector.selectedKeys().iterator();
                    while (selectionKeys.hasNext()) {
                        SelectionKey key = selectionKeys.next();
                        selectionKeys.remove();

                        if (key.isAcceptable()) {
                            SocketChannel socketChannel = serverSocketChannel.accept();
                            socketChannel.configureBlocking(false);
                            socketChannel.register(selector, SelectionKey.OP_READ);
                        }

                        if (key.isValid() && key.isReadable()) {
                            //当有读取事件发生时，从 SocketChannel 中读取文件信息，并创建独立的线程进行处理
                            SocketChannel socketChannel = (SocketChannel) key.channel();
                            scheduledExecutorService.execute(new Handle(socketChannel));
                            key.cancel();
                        }
                    }
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}
class Handler implements Runnable {
    private static final int BUFFER_SIZE = 1024 * 1024 * 64;

    SocketChannel socketChannel;

    public Handler(SocketChannel channel) {
        this.socketChannel = channel;
    }

    @Override
    public void run() {
        ByteBuffer fileNameLenBuffer = ByteBuffer.allocate(4);
        ByteBuffer fileSizeBuffer = ByteBuffer.allocate(8);
        try {
            //读取文件名字长度
            int read = socketChannel.read(fileNameLenBuffer);
            if (read == -1) {
                socketChannel.close();
                return;
            }
            fileNameLenBuffer.flip();
            int fileNameLen = fileNameLenBuffer.getInt();
            fileNameLenBuffer.clear();
            //读取文件名
            ByteBuffer fileNmaeBuffer = ByteBuffer.allocate(fileNameLen);
            read = socketChannel.read(fileNmaeBuffer);
            if (read == -1) {
                socketChannel.close();
                return;
            }
            fileNmaeBuffer.flip();
            byte[] fileNmaeBytes = new byte[fileNmaeBuffer.limit()];
            fileNmaeBuffer.get(fileNmaeBytes);
            String fileName = new String(fileNmaeBytes);

            //读取文件长度
            read = socketChannel.read(fileSizeBuffer);
            if (read == -1) {
                socketChannel.close();
                return;
            }
            fileSizeBuffer.flip();
            long fileSize = fileSizeBuffer.getLong();
            fileSizeBuffer.clear();


            //获取文件内容
            //创建文件目录和文件输出流
            File file = new File("E:\\ProjectWorker\\TestJob\\test_cursor\\etc\\" + fileName);
            if (!file.getParentFile().exists()) {
                file.getParentFile().mkdirs();
            }
            ByteBuffer dataBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
            ;
            try (RandomAccessFile fileOutputStream = new RandomAccessFile(file, "rw");
                 FileChannel fileChannel = fileOutputStream.getChannel()) {
                MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize);
                while (map.remaining() > 0) {
                    while (dataBuffer.remaining() > 0) {
                        read = socketChannel.read(dataBuffer);
                        if (read == -1) {
                            break;
                        }
                    }
                    dataBuffer.flip();
                    map.put(dataBuffer);
                    dataBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
                }
                System.out.println(System.currentTimeMillis() + ",File received: " + fileName + ", " + fileSize + " bytes.");
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                dataBuffer.clear();
                dataBuffer = null;
            }
            // 关闭连接
            socketChannel.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

发送端

package com.chun.zerocopy;

import java.io.File;
import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;

/**
 * @author chun
 * @date 2023/4/6 10:04
 */
public class FileSender {
    public static void main(String[] args) {

        ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(20);
        File file = new File("C:\\Users\\Downloads");
        File[] files = file.listFiles();
        for (File file1 : files) {
            if (!file1.isDirectory()) {
                scheduledExecutorService.execute(new Thread(()->{
                    send(file1);
                }));
            }
        }
        System.out.println(111);
//        System.exit(1);
    }

    private static void send(File file) {
        ByteBuffer allocate = ByteBuffer.allocate(1024);

        try {
            // 创建一个SocketChannel，连接到接收文件的服务
            SocketChannel socketChannel = SocketChannel.open();
            socketChannel.connect(new InetSocketAddress("127.0.0.1", 9999));
            long sTime = System.currentTimeMillis();
            // 获取SocketChannel的输出流
            FileInputStream fileInputStream = new FileInputStream(file);
            // 获取文件通道
            FileChannel fileChannel = fileInputStream.getChannel();
            // 使用mmap将文件映射到内存中
//            MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size());
            // 发送文件名和文件大小
            String fileName = file.getName();
            byte[] fileNameBytes = fileName.getBytes(StandardCharsets.UTF_8);
            long fileSize = file.length();
            allocate.putInt(fileNameBytes.length);
            allocate.put(fileNameBytes);
            allocate.putLong(fileSize);
            allocate.flip();
            socketChannel.write(allocate);
            // 使用零拷贝技术将文件发送给服务端
            long sPos = 0;
            while (sPos < fileSize) {
                sPos += fileChannel.transferTo(sPos, fileSize - sPos, socketChannel);
            }
            // 关闭流和socket
            fileChannel.close();
            fileInputStream.close();
            socketChannel.close();

            //记录时间
            System.out.println("fileName="+file.getName() + ",sendTime=" + (System.currentTimeMillis() - sTime));
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("fileName="+file.getName() + "e=" + e.getMessage());
        }
    }
}

测试

经测试300M数据0.5秒左右，可以同时传输大量文件