Kafka生产者介绍（八）：Kselector

Kselector使用NIO的方式实现I/O操作，用一个单独的线程管理多条网络的connect、read和write。主要字段有：

public class Selector implements Selectable {
    //java.nio.channels.Selector
    private final java.nio.channels.Selector nioSelector;
    //维护NodeId和KafkaChannel的映射关系，KafkaChannel是SocketChannel的封装，
    private final Map<String, KafkaChannel> channels;
    //已经完全发送出去的请求。
    private final List<Send> completedSends;
    //已经完全接受到的请求
    private final List<NetworkReceive> completedReceives;
    //暂存一次OP_READ时间过程中读到的所有请求。当一次READ事件处理完后，会把stagedReceives放入completeReceives中
    private final Map<KafkaChannel, Deque<NetworkReceive>> stagedReceives;
    //在调用SocketChannel#connect方法时立即完成的SelectionKey
    private final Set<SelectionKey> immediatelyConnectedKeys;
    //记录一次poll过程中发现的断开的连接和新建立的连接。
    private final List<String> disconnected;
    private final List<String> connected;
    // 向哪些node发送失败的请求，
    private final List<String> failedSends;
    private final Time time;
    private final SelectorMetrics sensors;
    private final String metricGrpPrefix;
    private final Map<String, String> metricTags;
    // 创建KafkaChannel的Builder
    private final ChannelBuilder channelBuilder;
    //各个链接的使用情况
    //一个连接太久没有用来执行读写操作，为了降低服务器端的压力，需要释放这些的连接。所以Selector有LRU机制，来淘汰这样的连接。
    private final Map<String, Long> lruConnections;
    private final long connectionsMaxIdleNanos;
    private final int maxReceiveSize;
    private final boolean metricsPerConnection;
    private long currentTimeNanos;
    private long nextIdleCloseCheckTime;
}

connect方法负责创建KafkaChannel，添加到channels集合中。

public class Selector implements Selectable {
    public void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException {
        if (this.channels.containsKey(id))
            throw new IllegalStateException("There is already a connection for id " + id);

        //创建socketChannel
        SocketChannel socketChannel = SocketChannel.open();
        //非阻塞模式
        socketChannel.configureBlocking(false);
        Socket socket = socketChannel.socket();
        //设置为长连接
        socket.setKeepAlive(true);
        //设置发送buffer的大小
        if (sendBufferSize != Selectable.USE_DEFAULT_BUFFER_SIZE)
            socket.setSendBufferSize(sendBufferSize);
        // receive的buffer大小
        if (receiveBufferSize != Selectable.USE_DEFAULT_BUFFER_SIZE)
            socket.setReceiveBufferSize(receiveBufferSize);
        socket.setTcpNoDelay(true);
        boolean connected;
        try {
            // 建立连接，非阻塞的，后面通过finishConnect方法确认连接是否建立
            connected = socketChannel.connect(address);
        } catch (UnresolvedAddressException e) {
            socketChannel.close();
            throw new IOException("Can't resolve address: " + address, e);
        } catch (IOException e) {
            socketChannel.close();
            throw e;
        }
        //把这个SocketChannel注册到nioSelector上，关注OP_CONNECT事件
        SelectionKey key = socketChannel.register(nioSelector, SelectionKey.OP_CONNECT);
        //创建KafkaChannel
        KafkaChannel channel = channelBuilder.buildChannel(id, key, maxReceiveSize);
        key.attach(channel);
        // nodeId和KafkaChannel放在channels中管理
        this.channels.put(id, channel);

        if (connected) {
            // OP_CONNECT won't trigger for immediately connected channels
            log.debug("Immediately connected to node {}", channel.id());
            immediatelyConnectedKeys.add(key);
            key.interestOps(0);
        }
    }
}

send方法把之前创建的RequestSend对象缓存到KafkaChannel的send字段中，并开始关注此连接的OP_WRITE事件，还没有发生网络I/O。在下次调用Selector的poll方法，才把RequestSend对象发送出去。
如果此时KafkaChannel中的send字段还保留一个没有完全发送成功的RequestSend请求，为了防止覆盖数据，就会抛异常，即一次poll过程中只能发送一次Send请求。
 

public class Selector implements Selectable {
    public void send(Send send) {
        KafkaChannel channel = channelOrFail(send.destination());
        try {
            //send字段还保留一个没有完全发送成功的RequestSend请求，则抛异常，放到failedSends中
            channel.setSend(send);
        } catch (CancelledKeyException e) {
            this.failedSends.add(send.destination());
            close(channel);
        }
    }
}

poll()是真正执行网路I/O的地方，它会调用nioSelector.select方法等待IO时间发生。当Channel可写时，发送send字段。Channel可读时，读取数据到KafkaChannel.receive中，读取到一个完整的NetworkReceivew后，会把它缓存到stageReceives中。
当一次pollSelectionKeys完成后把stageReceives的数据转移到completeReceives中

public class Selector implements Selectable {
    public void poll(long timeout) throws IOException {
        if (timeout < 0)
            throw new IllegalArgumentException("timeout should be >= 0");
        //清理掉上一次poll方法的结果
        clear();
        // 
        if (hasStagedReceives() || !immediatelyConnectedKeys.isEmpty())
            timeout = 0;

        /* check ready keys */
        long startSelect = time.nanoseconds();
        int readyKeys = select(timeout);
        long endSelect = time.nanoseconds();
        currentTimeNanos = endSelect;
        this.sensors.selectTime.record(endSelect - startSelect, time.milliseconds());

        if (readyKeys > 0 || !immediatelyConnectedKeys.isEmpty()) {
            //处理IO事件
            pollSelectionKeys(this.nioSelector.selectedKeys(), false);
            pollSelectionKeys(immediatelyConnectedKeys, true);
        }

        addToCompletedReceives();

        long endIo = time.nanoseconds();
        this.sensors.ioTime.record(endIo - endSelect, time.milliseconds());
        //关闭长期空闲的连接
        //在每次进行IO操作时，将Key：节点ID，Value：当前时间戳扔进哈希表里面，在IO操作进行完毕时，检查一下，最大的那个节点
        //它的最后一次IO时间+connectionsMaxIdleNanos(创建KafkaSelector时指定)，是否超过了当前的时间。 如果是，这个连接就会被关掉。
        maybeCloseOldestConnection();
    }
}

Selector#pollSelectionKeys是处理I/O的核心方法，处理connect、read和write事件。

public class Selector implements Selectable {
    private void pollSelectionKeys(Iterable<SelectionKey> selectionKeys, boolean isImmediatelyConnected) {
        Iterator<SelectionKey> iterator = selectionKeys.iterator();
        while (iterator.hasNext()) {
            SelectionKey key = iterator.next();
            iterator.remove();
            // 创建连接时注册到key上，在此获取
            KafkaChannel channel = channel(key);

            // register all per-connection metrics at once
            sensors.maybeRegisterConnectionMetrics(channel.id());
            //更新iru信息
            lruConnections.put(channel.id(), currentTimeNanos);

            try {

                // 对connect方法返回true或者OP_CONNETION方法的处理
                if (isImmediatelyConnected || key.isConnectable()) {
                    // finishConnect监测sockChannel是否建立完成，建立后，取消对CONNECT事件的关注，开始关注READ事件。
                    if (channel.finishConnect()) {
                        //添加到已连接的集合中。
                        this.connected.add(channel.id());
                        this.sensors.connectionCreated.record();
                    } else
                        //连接未完成，跳过。
                        continue;
                }

                // prepare进行身份验证。
                if (channel.isConnected() && !channel.ready())
                    channel.prepare();

                //read事件的处理
                if (channel.ready() && key.isReadable() && !hasStagedReceive(channel)) {
                    NetworkReceive networkReceive;
                    while ((networkReceive = channel.read()) != null)
                        //read读到一个完整的networkReceive，添加到StagedReceives中保存。
                        //读不到则返回null，下次处理READ事件时，继续读取。
                        addToStagedReceives(channel, networkReceive);
                }

                // 写事件处理
                if (channel.ready() && key.isWritable()) {
                    //channel.write()把send字段发送出去，未发送成功则返回null，完成则返回send，添加到completeSends中
                    Send send = channel.write();
                    if (send != null) {
                        this.completedSends.add(send);
                        this.sensors.recordBytesSent(channel.id(), send.size());
                    }
                }
                //completeSends和completeReceives表示在Selector端已经发送和接受到的请求，会在poll之后被不同的handleComplete调用。
                /* cancel any defunct sockets */
                if (!key.isValid()) {
                    close(channel);
                    this.disconnected.add(channel.id());
                }

            } catch (Exception e) {
                String desc = channel.socketDescription();
                if (e instanceof IOException)
                    log.debug("Connection with {} disconnected", desc, e);
                else
                    log.warn("Unexpected error from {}; closing connection", desc, e);
                close(channel);
                this.disconnected.add(channel.id());
            }
        }
    }
}

最后的读写操作由KafkaChannel完成
KafkaChannel主要字段有

public class KafkaChannel {
    //NodeId
    private final String id;
    //负责字节操作的传输层，KafkaChannel要操作SocketChannel时，都交给TransportLayer传输层去做,TransportLayer再使用底层的SocketChannel完成数据的操作。。
    private final TransportLayer transportLayer;
    private final Authenticator authenticator;
    private final int maxReceiveSize;
    //接收的数据
    private NetworkReceive receive;
    //发送的请求数据，一个KafkaChannel一次只存放一个请求数据。等着数据发送完成后，才能发送下一个请求数据。
    private Send send;
}

public class KafkaChannel {
    public void setSend(Send send) {
        if (this.send != null)
            throw new IllegalStateException("Attempt to begin a send operation with prior send operation still in progress.");
        //设置send字段
        this.send = send;
        this.transportLayer.addInterestOps(SelectionKey.OP_WRITE);
    }
    
    private boolean send(Send send) throws IOException {
        //如果send在一次write调用中没有发送法，SelectionKey的OP_WRITE事件没有取消，还会继续监听OP_WRITE事件，直到整个send字段完成。
        send.writeTo(transportLayer);
        // 判断ByteBuffer中是否还有剩余的字节，
        if (send.completed())
            transportLayer.removeInterestOps(SelectionKey.OP_WRITE);

        return send.completed();
    }
    
    public NetworkReceive read() throws IOException {
        NetworkReceive result = null;

        if (receive == null) {
            receive = new NetworkReceive(maxReceiveSize, id);
        }
        //receive从transportLayer中读取数据到NetworkReceive中。
        receive(receive);
        //直到读完整的一个networkReceive
        if (receive.complete()) {
            receive.payload().rewind();
            result = receive;
            receive = null;
        }
        return result;
    }                  
}