深入浅出Netty write

把数据返回客户端，需要经历三个步骤：

1. 申请一块缓存buf，写入数据。
2. 将buf保存到ChannelOutboundBuffer中。
3. 将ChannelOutboundBuffer中的buff输出到socketChannel中。

1. public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
2. ReferenceCountUtil.release(msg);
3.  
4. ByteBuf buf1 = ctx.alloc().buffer(4);
5. buf1.writeInt(1);
6.  
7. ByteBuf buf2 = ctx.alloc().buffer(4);
8. buf2.writeInt(2);
9.  
10. ByteBuf buf3 = ctx.alloc().buffer(4);
11. buf3.writeInt(3);
12.  
13. ctx.write(buf1);
14. ctx.write(buf2);
15. ctx.write(buf3);
16. ctx.flush();
17. }

为什么需要把buf保存到ChannelOutboundBuffer？

ctx.write()实现：

1. //AbstractChannelHandlerContext.java
2. public ChannelFuture write(Object msg) {
3. return write(msg, newPromise());
4. }
5.  
6. private void write(Object msg, boolean flush, ChannelPromise promise) {
7. AbstractChannelHandlerContext next = findContextOutbound();
8. EventExecutor executor = next.executor();
9. if (executor.inEventLoop()) {
10. next.invokeWrite(msg, promise);
11. if (flush) {
12. next.invokeFlush();
13. }
14. } else {
15. AbstractWriteTask task;
16. if (flush) {
17. task = WriteAndFlushTask.newInstance(next, msg, promise);
18. } else {
19. task = WriteTask.newInstance(next, msg, promise);
20. }
21. safeExecute(executor, task, promise, msg);
22. }
23. }

默认情况下，findContextOutbound()会找到pipeline的head节点，触发write方法。

1. //HeadContext.java
2. public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
3. unsafe.write(msg, promise);
4. }
5.  
6. //AbstractUnsafe
7. public final void write(Object msg, ChannelPromise promise) {
8. ChannelOutboundBuffer outboundBuffer = this.outboundBuffer;
9. if (outboundBuffer == null) {
10. safeSetFailure(promise, CLOSED_CHANNEL_EXCEPTION);
11. ReferenceCountUtil.release(msg);
12. return;
13. }
14.  
15. int size;
16. try {
17. msg = filterOutboundMessage(msg);
18. size = estimatorHandle().size(msg);
19. if (size < 0) {
20. size = 0;
21. }
22. } catch (Throwable t) {
23. safeSetFailure(promise, t);
24. ReferenceCountUtil.release(msg);
25. return;
26. }
27.  
28. outboundBuffer.addMessage(msg, size, promise);
29. }

outboundBuffer 随着Unsafe一起实例化，最终将msg通过outboundBuffer封装起来。

ChannelOutboundBuffer内部维护了一个Entry链表，并使用Entry封装msg。

1. unflushedEntry：指向链表头部
2. tailEntry：指向链表尾部
3. totalPendingSize：保存msg的字节数
4. unwritable：不可写标识

1. public void addMessage(Object msg, int size, ChannelPromise promise) {
2. Entry entry = Entry.newInstance(msg, size, total(msg), promise);
3. if (tailEntry == null) {
4. flushedEntry = null;
5. tailEntry = entry;
6. } else {
7. Entry tail = tailEntry;
8. tail.next = entry;
9. tailEntry = entry;
10. }
11. if (unflushedEntry == null) {
12. unflushedEntry = entry;
13. }
14.  
15. // increment pending bytes after adding message to the unflushed arrays.
16. // See https://github.com/netty/netty/issues/1619
17. incrementPendingOutboundBytes(size, false);
18. }

通过Entry.newInstance返回Entry实例，Netty对Entry采用了缓存策略，使用完的Entry实例需要清空并回收，难道是因为Entry实例化比较耗时？

新的entry默认插入链表尾部，并让tailEntry指向它。

1. private void incrementPendingOutboundBytes(long size, boolean invokeLater) {
2. if (size == 0) {
3. return;
4. }
5. long newWriteBufferSize = TOTAL_PENDING_SIZE_UPDATER.addAndGet(this, size);
6. if (newWriteBufferSize >= channel.config().getWriteBufferHighWaterMark()) {
7. setUnwritable(invokeLater);
8. }
9. }

方法incrementPendingOutboundBytes主要采用CAS更新totalPendingSize字段，并判断当前totalPendingSize是否超过阈值writeBufferHighWaterMark，默认是65536。如果totalPendingSize >= 65536，则采用CAS更新unwritable为1，并触发ChannelWritabilityChanged事件。

到此为止，全部的buf数据已经保存在outboundBuffer中。

ctx.flush()实现：

1. public ChannelHandlerContext flush() {
2. final AbstractChannelHandlerContext next = findContextOutbound();
3. EventExecutor executor = next.executor();
4. if (executor.inEventLoop()) {
5. next.invokeFlush();
6. } else {
7. Runnable task = next.invokeFlushTask;
8. if (task == null) {
9. next.invokeFlushTask = task = new Runnable() {
10. @Override
11. public void run() {
12. next.invokeFlush();
13. }
14. };
15. }
16. safeExecute(executor, task, channel().voidPromise(), null);
17. }
18.  
19. return this;
20. }

默认情况下，findContextOutbound()会找到pipeline的head节点，触发flush方法。

1. //HeadContext.java
2. public void flush(ChannelHandlerContext ctx) throws Exception {
3. unsafe.flush();
4. }
5.  
6. //AbstractUnsafe
7. public final void flush() {
8. assertEventLoop();
9. ChannelOutboundBuffer outboundBuffer = this.outboundBuffer;
10. if (outboundBuffer == null) {
11. return;
12. }
13. outboundBuffer.addFlush();
14. flush0();
15. }

方法addFlush主要对write过程添加的msg进行flush标识，其实我不清楚，这个标识过程有什么意义。

直接看flush0方法：

1. protected final void flush0() {
2. // Flush immediately only when there's no pending flush.
3. // If there's a pending flush operation, event loop will call forceFlush() later,
4. // and thus there's no need to call it now.
5. if (isFlushPending()) {
6. return;
7. }
8. super.flush0();
9. }
10.  
11. private boolean isFlushPending() {
12. SelectionKey selectionKey = selectionKey();
13. return selectionKey.isValid() && (selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0;
14. }

1. 如果当前selectionKey 是写事件，说明有线程执行flush过程，则直接返回。

2. 否则直接执行flush操作。
   ```
   protected void flush0() {
   if (inFlush0) {

   1. // Avoid re-entrance
   2. return;

   }

   final ChannelOutboundBuffer outboundBuffer = this.outboundBuffer;
   if (outboundBuffer == null || outboundBuffer.isEmpty()) {

   1. return;

   }

   inFlush0 = true;

   // Mark all pending write requests as failure if the channel is inactive.
   if (!isActive()) {

   1. try {
   2. if (isOpen()) {
   3. outboundBuffer.failFlushed(NOT_YET_CONNECTED_EXCEPTION, true);
   4. } else {
   5. // Do not trigger channelWritabilityChanged because the channel is closed already.
   6. outboundBuffer.failFlushed(CLOSED_CHANNEL_EXCEPTION, false);
   7. }
   8. } finally {
   9. inFlush0 = false;
   10. }
   11. return;

   }

   try {

   1. doWrite(outboundBuffer);

   } catch (Throwable t) {

   1. if (t instanceof IOException && config().isAutoClose()) {
   2. /**
   3. * Just call {@link #close(ChannelPromise, Throwable, boolean)} here which will take care of
   4. * failing all flushed messages and also ensure the actual close of the underlying transport
   5. * will happen before the promises are notified.
   6. *
   7. * This is needed as otherwise {@link #isActive()} , {@link #isOpen()} and {@link #isWritable()}
   8. * may still return {@code true} even if the channel should be closed as result of the exception.
   9. */
   10. close(voidPromise(), t, false);
   11. } else {
   12. outboundBuffer.failFlushed(t, true);
   13. }

   } finally {

   1. inFlush0 = false;

   }
   }

public boolean isActive() {
SocketChannel ch = javaChannel();
return ch.isOpen() && ch.isConnected();
}

1.  
2. 1. 如果当前socketChannel已经关闭，或断开连接，则执行失败操作。
3. 2. 否则执行doWrite把数据写入到socketChannel。

protected void doWrite(ChannelOutboundBuffer in) throws Exception {
for (;;) {
int size = in.size();
if (size == 0) {
// All written so clear OP_WRITE
clearOpWrite();
break;
}
long writtenBytes = 0;
boolean done = false;
boolean setOpWrite = false;

1. // Ensure the pending writes are made of ByteBufs only.
2. ByteBuffer[] nioBuffers = in.nioBuffers();
3. int nioBufferCnt = in.nioBufferCount();
4. long expectedWrittenBytes = in.nioBufferSize();
5. SocketChannel ch = javaChannel();
6.  
7. // Always us nioBuffers() to workaround data-corruption.
8. // See https://github.com/netty/netty/issues/2761
9. switch (nioBufferCnt) {
10. case 0:
11. // We have something else beside ByteBuffers to write so fallback to normal writes.
12. super.doWrite(in);
13. return;
14. case 1:
15. // Only one ByteBuf so use non-gathering write
16. ByteBuffer nioBuffer = nioBuffers[0];
17. for (int i = config().getWriteSpinCount() - 1; i >= 0; i --) {
18. final int localWrittenBytes = ch.write(nioBuffer);
19. if (localWrittenBytes == 0) {
20. setOpWrite = true;
21. break;
22. }
23. expectedWrittenBytes -= localWrittenBytes;
24. writtenBytes += localWrittenBytes;
25. if (expectedWrittenBytes == 0) {
26. done = true;
27. break;
28. }
29. }
30. break;
31. default:
32. for (int i = config().getWriteSpinCount() - 1; i >= 0; i --) {
33. final long localWrittenBytes = ch.write(nioBuffers, 0, nioBufferCnt);
34. if (localWrittenBytes == 0) {
35. setOpWrite = true;
36. break;
37. }
38. expectedWrittenBytes -= localWrittenBytes;
39. writtenBytes += localWrittenBytes;
40. if (expectedWrittenBytes == 0) {
41. done = true;
42. break;
43. }
44. }
45. break;
46. }
47.  
48. // Release the fully written buffers, and update the indexes of the partially written buffer.
49. in.removeBytes(writtenBytes);
50.  
51. if (!done) {
52. // Did not write all buffers completely.
53. incompleteWrite(setOpWrite);
54. break;