Netty使用案例 -堆外内存泄漏跟踪

内存池ByteBuf泄漏没有引起堆内存溢出

服务端代码入下

public class RouterServerHandler extends ChannelInboundHandlerAdapter {
    static ExecutorService executorService = Executors.newSingleThreadExecutor();
    PooledByteBufAllocator allocator = new PooledByteBufAllocator(false);


    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
        ByteBuf regMsg = (ByteBuf) msg;
        byte[] body = new byte[regMsg.readableBytes()];
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                ByteBuf respMsg = allocator.heapBuffer(body.length);
                //将请求直接转化为响应
                respMsg.writeBytes(body);
                ctx.writeAndFlush(respMsg);
                //ctx.fireChannelRead(msg);
            }
        });
    }
    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
        cause.printStackTrace();
        ctx.close();
    }
}

public class RouterServer {
    public static void main(String[] args) throws InterruptedException {
        EventLoopGroup bossGroup = new NioEventLoopGroup(1);
        EventLoopGroup workerGroup = new NioEventLoopGroup();


        try {
            ServerBootstrap serverBootstrap = new ServerBootstrap();
            serverBootstrap.group(bossGroup, workerGroup)
                    .channel(NioServerSocketChannel.class)
                    .option(ChannelOption.SO_BACKLOG, 100)
                    .handler(new LoggingHandler(LogLevel.INFO))
                    .childHandler(new ChannelInitializer<SocketChannel>() {
                        @Override
                        protected void initChannel(SocketChannel ch) throws Exception {
                            ChannelPipeline pipeline = ch.pipeline();
                            pipeline.addLast(new RouterServerHandler());
                        }
                    });

            //启动服务
            ChannelFuture f = serverBootstrap.bind(8080).sync();

            //阻塞知道服务关闭
            f.channel().closeFuture().sync();
        } finally {
            bossGroup.shutdownGracefully();
            workerGroup.shutdownGracefully();
        }
    }
}

客户端代码如下

public class RouterClinetHandler  extends ChannelInboundHandlerAdapter {
    private final ByteBuf firstMessage;


    public RouterClinetHandler() {
        System.out.println("RouterClinetHandler=");
        this.firstMessage = Unpooled.buffer(1024);
        for (int i = 0; i < firstMessage.capacity(); i++) {
            firstMessage.writeByte((byte)i);
        }
    }

    @Override
    public void channelActive(ChannelHandlerContext ctx) throws Exception {
        ctx.writeAndFlush(firstMessage);
    }

    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
        System.out.println("channelRead=");
        ctx.writeAndFlush(msg);
    }

    @Override
    public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
        ctx.flush();
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
        cause.printStackTrace();
        ctx.close();
    }
}

public class RouterClient {
    public static void main(String[] args) throws InterruptedException {
        //客户端访问服务
        EventLoopGroup group = new NioEventLoopGroup();
        try {

            Bootstrap b = new Bootstrap();
            b.group(group)
                    .channel(NioSocketChannel.class)
                    .option(ChannelOption.TCP_NODELAY, true)
                    .handler(new ChannelInitializer<SocketChannel>() {
                        @Override
                        protected void initChannel(SocketChannel ch) throws Exception {
                            ChannelPipeline p = ch.pipeline();
                            p.addLast(new RouterClinetHandler());
                        }
                    });
            // Start the client.
            ChannelFuture f = b.connect(String.valueOf(args[0]), Integer.valueOf(args[1])).sync();

            f.channel().closeFuture().sync();
        } finally {
            group.shutdownGracefully();
        }
    }
}

运行一段时间后server的效果

ByteBuf内存没有释放

对于ByteBuf申请相关代码进行排查，发现响应消息由业务线程创建，但却没有主动释放，因此是响应消息没有释放导致内存泄漏，但是图像看不出对内存泄漏。

PooledUnsafeHeapByteBuf并没有想象中的那么多，最多的是PooledUnsafeDirectByteBuf

业务内存快照

业务从内存池中申请了ByteBuf,但是没有主动释放，最后也没有发生内存泄漏，为什么？

如果是堆内存泄漏

//代码（AbstractNioByteChannel类）
@Override
    protected final Object filterOutboundMessage(Object msg) {
        if (msg instanceof ByteBuf) {
            ByteBuf buf = (ByteBuf) msg;
            if (buf.isDirect()) {
                return msg;
            }
            //这里创建DirectBuffer
            return newDirectBuffer(buf);
        }
        if (msg instanceof FileRegion) {
            return msg;
        }
        throw new UnsupportedOperationException(
                "unsupported message type: " + StringUtil.simpleClassName(msg) + EXPECTED_TYPES);
    }
    
//代码（AbstractNioChannel类）
protected final ByteBuf newDirectBuffer(ByteBuf buf) {
        final int readableBytes = buf.readableBytes();
        if (readableBytes == 0) {
            ReferenceCountUtil.safeRelease(buf);
            return Unpooled.EMPTY_BUFFER;
        }

        final ByteBufAllocator alloc = alloc();
        if (alloc.isDirectBufferPooled()) {
           //将HeapByteBuffer转换成DirectByteBuffer
            ByteBuf directBuf = alloc.directBuffer(readableBytes);
            directBuf.writeBytes(buf, buf.readerIndex(), readableBytes);
            //释放PooledHeapByteBuf到内存池
            ReferenceCountUtil.safeRelease(buf);
            return directBuf;
        }
   }
	。。。

如果消息完整地被写到SocketChannel中，则释放DirectByteBuffer，代码（ChannelOutboundBuffer类）：

public boolean remove() { 
    Entry e = flushedEntry; 
    if (e == null) { 
        clearNioBuffers(); 
        return false; 
    } 
    Object msg = e.msg; 
    ChannelPromise promise = e.promise; 
    int size = e.pendingSize; 
    removeEntry(e); 
    if (!e.cancelled) { 
        ReferenceCountUtil.safeRelease(msg); 
        safeSuccess(promise); 
        decrementPendingOutboundBytes(size, false, true); 
    } 
 //后续代码省略 
} 

代码跟踪

1. 设置断点到发送消息处writeAndFlush，获取到的PooledUnsafeHeapByteBuf实例的ID为2021

2. 在HeapByteBuffer转换成DirectByteBuffer处设置断点，发现实例ID为2021的PooledUnsafeHeapByteBuf被释放，在AbstractNioChannel.newDirectBuffer(buf)中的ReferenceCountUtil.safeRelease(buf)
   
3. 转换之后待发送的响应消息PooledUnsafeDirectByteBuf实例的ID为2049，AbstractNioChannel.newDirectBuffer中的ByteBuf directBuf = alloc.directBuffer(readableBytes);
   
4. 在响应消息发送完成后，实例ID为2049的PooledUnsafeDirectByteBuf被释放到内存池中
   以上跟踪确认不是响应消息没有主动释放导致的内存泄漏。

堆外内存泄漏

对请求消息的内存分配分析，发现NioByteUnsafe的read方法中申请了内存(NioByteUnsafe类)

//NioByteUnsaf.read
byteBuf = allocHandle.allocate(allocator); 
//继续对allocate方法进行分析，发现调用的是DefaultMaxMessagesRecvByteBuf- Allocator$MaxMessageHandle
public ByteBuf allocate(ByteBufAllocator alloc) { 
    return alloc.ioBuffer(guess()); 
} 
alloc.ioBuffer方法最终会调用PooledByteBufAllocator的newDirectBuffer方法创建PooledDirectByteBuf对象。

请求ByteBuf的创建分析完,由于业务的RouterServerHandler继承自ChannelInboundHandlerAdapter，它的channelRead(ChannelHandlerContext ctx, Object msg)方法执行完成，ChannelHandler的执行就结束了,请求ByteBuf被Netty框架申请后竟然没有被释放,
为了验证分析，在业务代码中调用ReferenceCountUtil的release方法进行手动对内存释放操作。代码调整添加

		ByteBuf regMsg = (ByteBuf) msg;
        byte[] body = new byte[regMsg.readableBytes()];
        //添加释放操作
        ReferenceCountUtil.release(regMsg);	
       。。。

修改之后继续进行压测，发现系统运行平稳，没有发生OutOfDirectMemoryError异常。对内存活动对象进行排序，没有再发现大量的PoolChunk对象，内存泄漏问题解决。

理解ByteBuf申请和释放

PooledDirectByteBuf和PooledHeapByteBuf：由Netty的NioEventLoop线程在处理Channel的读操作时分配，需要在业务ChannelInboundHandler处理完请求消息之后释放（通常在解码之后），它的释放有两种策略。

1. 策略1 ：业务ChannelInboundHandler继承自SimpleChannelInboundHandler，实现它的抽象方法channelRead0(ChannelHandlerContext ctx, I msg)，ByteBuf的释放业务不用关心，由SimpleChannelInboundHandler负责释放（SimpleChannelInboundHandler类）

public void channelRead(ChannelHandlerContext ctx, Object msg) throws
Exception { 
    boolean release = true; 
    try { 
        if (acceptInboundMessage(msg)) { 
            I imsg = (I) msg; 
            //调用channelRead0之后执行ReferenceCountUtil.release(msg)释放当前请求消息
            channelRead0(ctx, imsg); 
        } else { 
            release = false; 
            //如果没有匹配上需要继续执行后续的ChannelInboundHandler，则不释放当前请求消息，调用ctx.fireChannelRead(msg)驱动ChannelPipeline继续执行。
            ctx.fireChannelRead(msg); 
        } 
    } finally { 
        if (autoRelease && release) { 
            ReferenceCountUtil.release(msg); 
        } 
    } 
} 

将上边的Handler由继承的ChannelInboundHandlerAdapter调整SimpleChannelInboundHandler，对修改之后的代码做性能测试，发现内存占用平稳，无内存泄漏问题。

public class SimpServerHandler extends SimpleChannelInboundHandler<ByteBuf> {
    static ExecutorService executorService = Executors.newSingleThreadExecutor();
    PooledByteBufAllocator allocator = new PooledByteBufAllocator(false);
    @Override
    protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception {
        ByteBuf regMsg = (ByteBuf) msg;
        byte[] body = new byte[msg.readableBytes()];
        //注意这里删除要不会出现onUnhandledInboundException
        //ReferenceCountUtil.release(regMsg);
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                ByteBuf respMsg = allocator.heapBuffer(body.length);
                //将请求直接转化为响应
                respMsg.writeBytes(body);
                ctx.writeAndFlush(respMsg);
                //ctx.fireChannelRead(msg);
            }
        });
    }
}

2. 策略2 在业务ChannelInboundHandler中调用ctx.fireChannelRead(msg)方法，让请求消息继续向后执行，直到调用DefaultChannelPipeline的内部类TailContext，由它来负责释放请求消息。

protected void onUnhandledInboundMessage(Object msg) { 
    try { 
        logger.debug( 
        "Discarded inbound message {} that reached at the tail of the pipeline. " + 
        "Please check your pipeline configuration.", msg); 
    } finally { 
        ReferenceCountUtil.release(msg); 
    } 
} 

//调整之前的代码RouterServerHandler
 //ctx.writeAndFlush(respMsg);修改为以下
 ctx.fireChannelRead(msg);

同样使用后运行正常。

基于ByteBuf请求和响应创建

基于非内存请求

//业务使用非内存池模式覆盖Netty默认的内存池模式创建请求ByteBuf，按照内存池的方式释放内存
.childHandler(new ChannelInitializer<SocketChannel>() { 
    @Override 
    public void initChannel(SocketChannel ch) throws Exception { 
        ChannelPipeline p = ch.pipeline(); 
        //设置非内存池请求
        ch.config().setAllocator(UnpooledByteBufAllocator.DEFAULT);
        p.addLast(new RouterServerHandler()); 
    } 
 }); 
} 

基于内存池响应

只要调用了writeAndFlush或者flush方法，在消息发送完成后都会由Netty框架进行内存释放，业务不需要主动释放内存。

基于非内存池响应

无论是基于内存池还是非内存池分配的ByteBuf，如果是堆内存，则将堆内存转换成堆外内存，然后释放HeapByteBuffer，待消息发送完成，再释放转换后的DirectByteBuf；如果是DirectByteBuffer，则不需要转换，待消息发送完成之后释放。因此对于需要发送的响应ByteBuf，由业务创建，但是不需要由业务来释放。

解读《Netty进阶之路：跟着案例学Netty》-Netty内存池泄漏疑云案例