netty 管道和handler的加载和处理流程

一、pom引入包，此处版本为4.1.52.Final

   <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-native-kqueue</artifactId>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport-native-epoll</artifactId>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-common</artifactId>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-codec</artifactId>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-buffer</artifactId>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-transport</artifactId>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-resolver-dns</artifactId>
        </dependency>
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-handler</artifactId>
        </dependency>

二、简单的实体对象利用NETTY进行序列化和数据传输的示例，服务端代码

1.启动类

  public boolean bind(){
        EventLoopGroup bossEventLoopGroup = new NioEventLoopGroup(new NamedThreadFactory("bossThread",false));
        EventLoopGroup workEventLoopGroup = new NioEventLoopGroup(new NamedThreadFactory("workThread",false));
        ServerBootstrap bootstrap = new ServerBootstrap();
        bootstrap.group(bossEventLoopGroup,workEventLoopGroup)
                .channel(NioServerSocketChannel.class).option(ChannelOption.SO_BACKLOG,128)
                .handler(new LoggingHandler(LogLevel.INFO))
                .childHandler( new PojoServerIntitlizer());
        try {
            log.debug(" will start server");
            ChannelFuture closeFuture =  bootstrap.bind(port).sync();
            log.debug("  server is closing");
            closeFuture.channel().closeFuture().sync();
            log.debug("  server is closed");
        } catch (InterruptedException e) {
            e.printStackTrace();
        }finally {
            bossEventLoopGroup.shutdownGracefully();
            workEventLoopGroup.shutdownGracefully();
            log.debug("  release event loop group");
        }
        return true;
    }

2.管道中的handler初始化类

@Slf4j
public class PojoServerIntitlizer extends ChannelInitializer<SocketChannel> {

    @Override
    protected void initChannel(SocketChannel ch) throws Exception {
        ch.pipeline().addLast(new PojoEncoder());
        ch.pipeline().addLast(new PojoJsonEncoder());
        ch.pipeline().addLast(new PojoDecoder());
        ch.pipeline().addLast(new PojoServerHandler());
        log.debug(" initChannel handler names:{}",ch.pipeline().names());
    }
}

initChannel handler names:[PojoClientIntitlizer#0, PojoEncoder#0, PojoJsonEncoder#0, PojoDecoder#0, PojoClientHandler#0, DefaultChannelPipeline$TailContext#0]

3.POJO对象转换JSON编码类

@Slf4j
public class PojoJsonEncoder extends MessageToMessageEncoder {

    @Override
    protected void encode(ChannelHandlerContext ctx, Object msg, List out) throws Exception {
        String msgStr = JSONUtil.toJsonStr(msg);
        log.debug(" encode json data. msgStr:{}",msgStr);
        out.add(msgStr);
    }


}

4.POJO转换为JSON后将字符串转为带长度的字节流编码类

@Slf4j
public class PojoEncoder extends MessageToByteEncoder {

    @Override
    protected void encode(ChannelHandlerContext ctx, Object msg, ByteBuf out) throws Exception {
          byte[] msgByte = ((String)msg).getBytes(StandardCharsets.UTF_8);
          int msgLen = msgByte.length;
          log.debug(" encode byte data. msgLen:{}",msgLen);
          out.writeInt(msgLen);
          out.writeBytes(msgByte);

    }
}

5.输入字节流转换为POJO对象类

@Slf4j
public class PojoDecoder extends ByteToMessageDecoder {
    private static final int HEAD_LEN =4;
    @Override
    protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List<Object> list) throws Exception {
           log.debug(" decode data. read len:{}",byteBuf.readableBytes());
           if (byteBuf.readableBytes() < HEAD_LEN){
               return;
           }
           byteBuf.markReaderIndex();
           int dataLen = byteBuf.readInt();
           if (dataLen <= 0){
               byteBuf.resetReaderIndex();
               return;
           }
           if (byteBuf.readableBytes() < dataLen){
               byteBuf.resetReaderIndex();
               return;
           }
           byte[] data = new byte[byteBuf.readableBytes()];
           byteBuf.readBytes(data);
           Object obj = JSONUtil.toBean(new String(data,"UTF-8"), NettyConstant.pojoCls);
           list.add(obj);
    }
}

6.业务读处理类

@Slf4j
public class PojoServerHandler extends ChannelInboundHandlerAdapter {
    @Override
    public void channelActive(ChannelHandlerContext ctx) throws Exception {
//        super.channelActive(ctx);
        log.debug(" channelRegistered name:{},localAddress:{},remoteAddress:{} threadName:{}"
                ,ctx.name(),ctx.channel().localAddress(),ctx.channel().remoteAddress()
                ,Thread.currentThread().getName());
        TelnetMsgDto telnetMsgDto = TelnetMsgDto.builder().nick("游客").sendTime(DateUtil.currentSeconds())
                .content("欢迎你 \r\n").type(PojoContentType.POJO_CONTENT_TYPE_MSG).build();
        ctx.writeAndFlush(telnetMsgDto);
    }

    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
//        super.channelRead(ctx, msg);
        TelnetMsgDto telnetMsgDto = (TelnetMsgDto) msg;
        log.debug(" channelRead telnetMsgDto:{}",telnetMsgDto);
        TelnetMsgDto sendMsgDtO = null;
        switch (((TelnetMsgDto) msg).getType()){
            case POJO_CONTENT_TYPE_LOGIN:
                 sendMsgDtO = TelnetMsgDto.builder().nick(telnetMsgDto.getNick()).sendTime(DateUtil.currentSeconds())
                        .content(" 欢迎你 \r\n").type(PojoContentType.POJO_CONTENT_TYPE_MSG).build();

                break;
            case POJO_CONTENT_TYPE_MSG:
                 sendMsgDtO = TelnetMsgDto.builder().nick(telnetMsgDto.getNick()).sendTime(DateUtil.currentSeconds())
                        .content(" 回复:" + telnetMsgDto.getContent() + "\r\n")
                         .type(PojoContentType.POJO_CONTENT_TYPE_MSG).build();
                break;
            case POJO_CONTENT_TYPE_LOGOUT:
                sendMsgDtO = TelnetMsgDto.builder().nick(telnetMsgDto.getNick()).sendTime(DateUtil.currentSeconds())
                        .content(" bye byte:" + telnetMsgDto.getNick() + "\r\n")
                        .type(PojoContentType.POJO_CONTENT_TYPE_MSG).build();
                break;
            default:
                throw new IllegalStateException("Unexpected value: " + ((TelnetMsgDto) msg).getType());
        }
        if (ObjectUtil.isNotNull(sendMsgDtO)){
            ctx.writeAndFlush(sendMsgDtO);
        }
    }

三、客户端类

  1.启动类

@Slf4j
public class PojoClientServer {
    private int port;
    private String host;
    private Channel channel;

    public PojoClientServer(int port, String host) {
        this.port = port;
        this.host = host;
    }

    public ChannelFuture sendData(TelnetMsgDto telnetMsgDto){
        ChannelFuture channelFuture = this.channel.writeAndFlush(telnetMsgDto);
        return channelFuture;
    }

    public boolean connect(){
        EventLoopGroup workEventLoopGroup = new NioEventLoopGroup(new NamedThreadFactory("clientThread",false));
        Bootstrap bootstrap = new Bootstrap();
        bootstrap.group(workEventLoopGroup)
                .channel(NioSocketChannel.class)
                .handler(new PojoClientIntitlizer());
        try {
            log.debug(" will start connect server");
            this.channel  =  bootstrap.connect(host,port).sync().channel();
            log.debug("  client is START");
            String nick = "jack";
            this.sendData(TelnetMsgDto.buildLoginMsg(nick));

            String sendData[] = {"hello","world","qiutian","happy","bye"};
//            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(System.in));
            ChannelFuture sendFuture = null;
            int i = 0;
            while (true){
//                String inputData = bufferedReader.readLine() ;
                if (i >= sendData.length){
                    sendFuture = this.sendData(TelnetMsgDto.buildLogoutMsg(nick));
                    break;
                }
                String inputData = sendData[i] ;
                i++;
                sendFuture = this.sendData(TelnetMsgDto.buildNormalMsg(nick,inputData));
                log.debug("  send data inputData:{},i:{}",inputData,i);
                if (ObjectUtil.isNotNull(sendFuture)){
                    sendFuture.sync();
                }
                TimeUnit.SECONDS.sleep(3);
              /*  if (StrUtil.contains(inputData,"bye")){
                    log.debug("  end.");
                    this.channel.close().sync();
                    break;
                }*/
            }
            TimeUnit.SECONDS.sleep(3);
            log.debug("  client is closed");
            this.channel.close().sync();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            workEventLoopGroup.shutdownGracefully();
            log.debug("  release event loop group");
        }
        return true;
    }
}

2.HANDLER初始化加入管道类

@Slf4j
public class PojoClientIntitlizer extends ChannelInitializer<SocketChannel> {


    @Override
    protected void initChannel(SocketChannel ch) throws Exception {
        ch.pipeline().addLast(new PojoEncoder());
        ch.pipeline().addLast(new PojoJsonEncoder());
        ch.pipeline().addLast(new PojoDecoder());
        ch.pipeline().addLast(new PojoClientHandler());

        log.debug(" initChannel handler names:{}",ch.pipeline().names());
    }
}

3.客户端读处理类

@Slf4j
public class PojoClientHandler extends SimpleChannelInboundHandler<TelnetMsgDto> {


    @Override
    protected void channelRead0(ChannelHandlerContext ctx, TelnetMsgDto msg) throws Exception {
        log.debug(" channelRead0 msg:{}",msg);
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
//        super.exceptionCaught(ctx, cause);
        cause.printStackTrace();
        ctx.close();
    }
}

四、客户端发送数据的处理类流程

  1.客户端向channel写入数据

    public ChannelFuture sendData(TelnetMsgDto telnetMsgDto){
        ChannelFuture channelFuture = this.channel.writeAndFlush(telnetMsgDto);
        return channelFuture;
    }

2.channel.writeAndFlush会调用pipeline.writeAndFlush,然后会调用到tail.writeAndFlush(msg)

   io.netty.channel.DefaultChannelPipeline
 public final ChannelFuture writeAndFlush(Object msg) {
        return tail.writeAndFlush(msg);
    }

3.这里的tail为管道默认生成的最后一个HANDLER,实际上为AbstractChannelHandlerContext

 io.netty.channel.AbstractChannelHandlerContext
private void write(Object msg, boolean flush, ChannelPromise promise) {
        final AbstractChannelHandlerContext next = findContextOutbound(flush ?
                (MASK_WRITE | MASK_FLUSH) : MASK_WRITE);
        final Object m = pipeline.touch(msg, next);
        EventExecutor executor = next.executor();
        if (executor.inEventLoop()) {
            if (flush) {
                next.invokeWriteAndFlush(m, promise);
            } else {
                next.invokeWrite(m, promise);
            }
        } else {
            final WriteTask task = WriteTask.newInstance(next, m, promise, flush);
            if (!safeExecute(executor, task, promise, m, !flush)) {
                // We failed to submit the WriteTask. We need to cancel it so we decrement the pending bytes
                // and put it back in the Recycler for re-use later.
                //
                // See https://github.com/netty/netty/issues/8343.
                task.cancel();
            }
        }
    }

,注意，这里有一个找输出bound的过程。它是从HANDLER处理链中的尾部向前找，找到一个仅仅可以处理输出bound的HANDLE进行处理。所以是我们初始化管理处理器时，outbound的执行顺序是依次从尾部向头部找，逐个执行。

    private AbstractChannelHandlerContext findContextOutbound(int mask) {
        AbstractChannelHandlerContext ctx = this;
        EventExecutor currentExecutor = executor();
        do {
            ctx = ctx.prev;
        } while (skipContext(ctx, currentExecutor, mask, MASK_ONLY_OUTBOUND));
        return ctx;
    }

这里面的executor实际上就是一个NioEventLoop,next为ChannelHandlerContext(PojoJsonEncoder#0, [id: 0x08b5bc88, L:/127.0.0.1:52977 - R:/127.0.0.1:7110])

 4.由于不是在NioEventLoop线程进行发送数据，为了避免数据竞争，所以都会提交任务到NioEventLoop进行处理。这里构造一个WriteTask异步任务。

 5.现在我们看到异步执行了writeTask的任务

    WriteTask的执行函数
@Override
        public void run() {
            try {
                decrementPendingOutboundBytes();
                if (size >= 0) {
                    ctx.invokeWrite(msg, promise);
                } else {
                    ctx.invokeWriteAndFlush(msg, promise);
                }
            } finally {
                recycle();
            }
        }

6.接着执行了AbstractChannelHandlerContext.invokeWrite0

  private void invokeWrite0(Object msg, ChannelPromise promise) {
        try {
            ((ChannelOutboundHandler) handler()).write(this, msg, promise);
        } catch (Throwable t) {
            notifyOutboundHandlerException(t, promise);
        }
    }

7.这就是调用处理HANDLER的write方法，由于PojoJsonEncoder继承于MessageToMessageEncoder，所以要在MessageToMessageEncoder的write方法里面再调用encode进行消息转换。

 public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
        CodecOutputList out = null;
        try {
            if (acceptOutboundMessage(msg)) {
                out = CodecOutputList.newInstance();
                I cast = (I) msg;
                encode(ctx, cast, out);

            } else {
                ctx.write(msg, promise);
            }
        } catch (EncoderException e) {
            throw e;
        } catch (Throwable t) {
            throw new EncoderException(t);
        } finally {
            if (out != null) {
                try {
                    final int sizeMinusOne = out.size() - 1;
                    if (sizeMinusOne == 0) {
                        ctx.write(out.getUnsafe(0), promise);
                    } else if (sizeMinusOne > 0) {
                        if (promise == ctx.voidPromise()) {
                            writeVoidPromise(ctx, out);
                        } else {
                            writePromiseCombiner(ctx, out, promise);
                        }
                    }
                } finally {
                    out.recycle();
                }
            }
        }
    }

8.消息转换完毕后放到一个名为out 的list里面，然后在上述的write方法中发现out不为空，则再次触发ctx.write(out.getUnsafe(0), promise);注意这里跟第一次tail的write方法一样，又会从当前(PojoJsonEncoder)的HANDLER中向前找下一个可支持outBound的HANDLER类，所以就会找到PojoEncoder

9.接着会调用到next.write方法，由于此次是在同一个NioEventLoop，所以直接调用， PojoEncoder继承于MessageToByteEncoder，所以跟上面那个MessageToMessageEncoder类似，会先调用此类的write方法，方法内部调用encode将消息体转换为字节流。

 MessageToByteEncoder
 public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
        ByteBuf buf = null;
        try {
            if (acceptOutboundMessage(msg)) {
                @SuppressWarnings("unchecked")
                I cast = (I) msg;
                buf = allocateBuffer(ctx, cast, preferDirect);
                try {
                    encode(ctx, cast, buf);
                } finally {
                    ReferenceCountUtil.release(cast);
                }

                if (buf.isReadable()) {
                    ctx.write(buf, promise);
                } else {
                    buf.release();
                    ctx.write(Unpooled.EMPTY_BUFFER, promise);
                }
                buf = null;
            } else {
                ctx.write(msg, promise);
            }
        } catch (EncoderException e) {
            throw e;
        } catch (Throwable e) {
            throw new EncoderException(e);
        } finally {
            if (buf != null) {
                buf.release();
            }
        }
    }

10.转换完后，会再次调用ctx.write(buf, promise)，这就是一个责任链模式，这次找到的next为head节点，ChannelHandlerContext(DefaultChannelPipeline$HeadContext#0, [id: 0x7cb49772, L:/127.0.0.1:50203 - R:/127.0.0.1:7110])

 11.由于在同一个NioEventLoop,所以直接调用write，最终调用到HeadContext.write，这里为终点了，没有再调用ctx.write

    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) {
            unsafe.write(msg, promise);
        }

这个unsafe为NioSocketChannel内部的safe,这里面就是NIO的SOCKET封装类了，这个write只是写入缓冲区，当调用flush时再会真正进行网络发送。 

12.unsafe为NioSocketChannelUnsafe,这个类内部有一个ChannelOutboundBuffer outboundBuffer的成员变量，用来保存待发送缓存区。write方法将将数据保存到此缓存区。

    protected abstract class AbstractUnsafe implements Unsafe {

        private volatile ChannelOutboundBuffer outboundBuffer = new ChannelOutboundBuffer(AbstractChannel.this);

@Override
        public final void write(Object msg, ChannelPromise promise) {
            ChannelOutboundBuffer outboundBuffer = this.outboundBuffer;
            outboundBuffer.addMessage(msg, size, promise);
        }

调用堆栈

 13.ctx.flush函数调用流程，我们简单描述。最终会调用到NioSocketChannel.doWrite方法，这个里面会调用java的java.nio.channels.SocketChannel[connected local=/127.0.0.1:50203 remote=/127.0.0.1:7110]，进行写入数据，真正发送。

NioSocketChannel 
protected void doWrite(ChannelOutboundBuffer in) throws Exception {
        SocketChannel ch = javaChannel();
        int writeSpinCount = config().getWriteSpinCount();
        do {
            int nioBufferCnt = in.nioBufferCount();
            switch (nioBufferCnt) {
                case 1: {
                    ByteBuffer buffer = nioBuffers[0];
                    int attemptedBytes = buffer.remaining();
                    final int localWrittenBytes = ch.write(buffer);
                    if (localWrittenBytes <= 0) {
                        incompleteWrite(true);
                        return;
                    }
                    break;
                }
                }
            }
        } while (writeSpinCount > 0);
    }

 14.至此，数据写入发送流程就结束了。

 15.注意：ctx.write和ctx.channel.write是有的区别的。ctx.write是从当前handler中向前找一个可写的handler然后调用其write函数，ctx.channel.write则是调用pipline.write，这个方法，会是从整个管道的尾部向前找一个可写的handler进行处理，会比前一个更耗时。

五、客户端接收读取数据的流程。

  1.目前是由NioEventLoop的run方法，会循环检测网络IO的select事件，processSelectedKeysOptimized，注意，这里的selectionKey有一个attachment附加对象，就是我们的NioSocketChannel

 private void processSelectedKeysOptimized() {
        for (int i = 0; i < selectedKeys.size; ++i) {
            final SelectionKey k = selectedKeys.keys[i];
            selectedKeys.keys[i] = null;
            final Object a = k.attachment();
            if (a instanceof AbstractNioChannel) {
                processSelectedKey(k, (AbstractNioChannel) a);
            } 
        }
    }

2.调用NioEventLoop.processSelectedKey检测socket的连接，可写，可读事件，当在SOCKETCHANNEL发现数据可读时，会调用unsafe.read.

  private void processSelectedKey(SelectionKey k, AbstractNioChannel ch) {
        final AbstractNioChannel.NioUnsafe unsafe = ch.unsafe();
        try {
            int readyOps = k.readyOps();
            if ((readyOps & SelectionKey.OP_CONNECT) != 0) {
                int ops = k.interestOps();
                ops &= ~SelectionKey.OP_CONNECT;
                k.interestOps(ops);
                unsafe.finishConnect();
            }

            if ((readyOps & SelectionKey.OP_WRITE) != 0) {
                // Call forceFlush which will also take care of clear the OP_WRITE once there is nothing left to write
                ch.unsafe().forceFlush();
            }


            if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) {
                unsafe.read();
            }
        } catch (CancelledKeyException ignored) {
            unsafe.close(unsafe.voidPromise());
        }
    }

unsafe跟上面写时的对象一样为NioSocketChannel的内部safe，封装了原始的socketChannel

 3.unsafe.read函数就会从网络读取数据，然后调用pipeline.fireChannelRead(byteBuf)，向管道发送读事件。

AbstractNioByteChannel    
@Override
        public final void read() {
            final ChannelConfig config = config();
            final ChannelPipeline pipeline = pipeline();
            final ByteBufAllocator allocator = config.getAllocator();
            final RecvByteBufAllocator.Handle allocHandle = recvBufAllocHandle();
            allocHandle.reset(config);

            ByteBuf byteBuf = null;
            boolean close = false;
            try {
                do {
                    byteBuf = allocHandle.allocate(allocator);
                    allocHandle.lastBytesRead(doReadBytes(byteBuf));
                    allocHandle.incMessagesRead(1);
                    readPending = false;
                    pipeline.fireChannelRead(byteBuf);
                    byteBuf = null;
                } while (allocHandle.continueReading());

                allocHandle.readComplete();
                pipeline.fireChannelReadComplete();
            } 
        }
    }

4.pipeline.fireChannelRead(byteBuf)，直接调用头节点的write方法。

DefaultChannelPipeline    
public final ChannelPipeline fireChannelRead(Object msg) {
        AbstractChannelHandlerContext.invokeChannelRead(head, msg);
        return this;
    }

5.head节点的write方法,从head节点向尾部找一个支持inbound的HANDLER进行处理，刚好跟写入相反

   AbstractChannelHandlerContext
 public ChannelHandlerContext fireChannelRead(final Object msg) {
        invokeChannelRead(findContextInbound(MASK_CHANNEL_READ), msg);
        return this;
    }

    private AbstractChannelHandlerContext findContextInbound(int mask) {
        AbstractChannelHandlerContext ctx = this;
        EventExecutor currentExecutor = executor();
        do {
            ctx = ctx.next;
        } while (skipContext(ctx, currentExecutor, mask, MASK_ONLY_INBOUND));
        return ctx;
    }

6.找到的第一个handler为ChannelHandlerContext(PojoDecoder#0, [id: 0x7cb49772, L:/127.0.0.1:50203 - R:/127.0.0.1:7110])

 7.由于PojoDecoder继承于ByteToMessageDecoder类，所以会调用ByteToMessageDecoder.

channelRead，在此方法内尝试解码转换。

  public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
        if (msg instanceof ByteBuf) {
            CodecOutputList out = CodecOutputList.newInstance();
            try {
                first = cumulation == null;
                cumulation = cumulator.cumulate(ctx.alloc(),
                        first ? Unpooled.EMPTY_BUFFER : cumulation, (ByteBuf) msg);
                callDecode(ctx, cumulation, out);
            } catch (DecoderException e) {
                throw e;
            } catch (Exception e) {
                throw new DecoderException(e);
            } finally {
                try {
                    int size = out.size();
                    firedChannelRead |= out.insertSinceRecycled();
                    fireChannelRead(ctx, out, size);
                } finally {
                    out.recycle();
                }
            }
        } else {
            ctx.fireChannelRead(msg);
        }
    }

8.转换成功后，会调用fireChannelRead,对out这个list列表进行逐个循环调用，ctx.fireChannelRead(msgs.getUnsafe(i));，通知给下一个inBound的handler.

   static void fireChannelRead(ChannelHandlerContext ctx, CodecOutputList msgs, int numElements) {
        for (int i = 0; i < numElements; i ++) {
            ctx.fireChannelRead(msgs.getUnsafe(i));
        }
    }

9.这次找到的下一个handler为PojoClientHandler，由于此类继承SimpleChannelInboundHandler,

最终调到其read0方法。这次可以看到输出整个对象。

 10.可以看对象的数据了，至此读数据流程结束了。