Netty源码（二）EventLoop的创建

EventLoop

因为最常用的就是NioEventLoop，所以主要看下NioEventLoop的实现

类继承结构

EventExecutor

EventExecutor的作用主要分为两个部分：

1. 一部分来自AbstractExecutorService，因此具有提交任务，运行任务的能力
2. 另一部分的能力是自己定义的，用来判断一个线程是否在EventLoop中
   下面看下EventExecutor自己定义了哪些方法

public interface EventExecutor extends EventExecutorGroup {

    /**
     * Returns a reference to itself.
     */
     // 一直返回自己
    @Override
    EventExecutor next();

    /**
     * Return the {@link EventExecutorGroup} which is the parent of this {@link EventExecutor},
     */
     // 返回当前EventExecutor属于的EventExecutorGroup
    EventExecutorGroup parent();

    /**
     * Calls {@link #inEventLoop(Thread)} with {@link Thread#currentThread()} as argument
     */
     // 判断当前线程是否在eventLoop中
    boolean inEventLoop();

    /**
     * Return {@code true} if the given {@link Thread} is executed in the event loop,
     * {@code false} otherwise.
     */
    // 判断指定线程是否在eventLoop中
    boolean inEventLoop(Thread thread);

    /**
     * Return a new {@link Promise}.
     */
    <V> Promise<V> newPromise();

    /**
     * Create a new {@link ProgressivePromise}.
     */
    <V> ProgressivePromise<V> newProgressivePromise();

    /**
     * Create a new {@link Future} which is marked as successes already. So {@link Future#isSuccess()}
     * will return {@code true}. All {@link FutureListener} added to it will be notified directly. Also
     * every call of blocking methods will just return without blocking.
     */
    <V> Future<V> newSucceededFuture(V result);

    /**
     * Create a new {@link Future} which is marked as fakued already. So {@link Future#isSuccess()}
     * will return {@code false}. All {@link FutureListener} added to it will be notified directly. Also
     * every call of blocking methods will just return without blocking.
     */
    <V> Future<V> newFailedFuture(Throwable cause);
}

OrderedEventExecutor

EventExecutor的标记接口，会按照一定的顺序来执行提交的任务

public interface OrderedEventExecutor extends EventExecutor {
}

AbstractEventExecutor

AbstractEventExecutor是EventExecutor的一个抽象类，实现了部分方法

重要属性

// 所属的ExecutorGroup
private final EventExecutorGroup parent;
// eventExecutor集合，只有自己一个
private final Collection<EventExecutor> selfCollection = Collections.<EventExecutor>singleton(this);

next

当调用next方法来获取一个EventExecutor时，会一直返回自身

@Override
public EventExecutor next() {
    return this;
}

inEventLoop

当调用inEventLoop判断当前线程是否在eventLoop中时，会调用子类的实现

@Override
public boolean inEventLoop() {
    return inEventLoop(Thread.currentThread());
}

submit

会调用父类AbstractEventExecutor的submit

@Override
public Future<?> submit(Runnable task) {
    return (Future<?>) super.submit(task);
}

schedule

不支持schedule

@Override
public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
    throw new UnsupportedOperationException();
}

iterator

返回用来遍历eventExecutor的迭代器

@Override
public Iterator<EventExecutor> iterator() {
   return selfCollection.iterator();
}

newXXXPromise

Promise是特殊的Future，主要是支持手动设置成功结果和失败原因
可以看到下面创建的Promise都会将当前的eventExecutor实例传入

@Override
public <V> Promise<V> newPromise() {
    return new DefaultPromise<V>(this);
}

@Override
public <V> ProgressivePromise<V> newProgressivePromise() {
    return new DefaultProgressivePromise<V>(this);
}

newXXXFuture

可以看到下面创建的Future都会将当前eventExecutor传入

@Override
public <V> Future<V> newSucceededFuture(V result) {
    return new SucceededFuture<V>(this, result);
}

@Override
public <V> Future<V> newFailedFuture(Throwable cause) {
    return new FailedFuture<V>(this, cause);
}

newTaskFor

在创建PromiseTask的时候，会将当前的executor作为参数传入

@Override
protected final <T> RunnableFuture<T> newTaskFor(Runnable runnable, T value) {
    return new PromiseTask<T>(this, runnable, value);
}

@Override
protected final <T> RunnableFuture<T> newTaskFor(Callable<T> callable) {
    return new PromiseTask<T>(this, callable);
}

SingleThreadEventExecutor

AbstractEventExecutor的一个子类，其子类的实现类在一个线程中来执行所有提交的任务

重要属性

// 任务队列中最多pending任务个数
static final int DEFAULT_MAX_PENDING_EXECUTOR_TASKS = Math.max(16,
            SystemPropertyUtil.getInt("io.netty.eventexecutor.maxPendingTasks", Integer.MAX_VALUE));
// 当前状态
// 未启动
private static final int ST_NOT_STARTED = 1;
// 已启动
private static final int ST_STARTED = 2;
// 正在关闭
private static final int ST_SHUTTING_DOWN = 3;
// 已关闭
private static final int ST_SHUTDOWN = 4;
// 已终结
private static final int ST_TERMINATED = 5;

// 任务队列
private final Queue<Runnable> taskQueue;
// 执行任务的线程
private volatile Thread thread;
// 执行器
private final Executor executor;
// 最多pending任务个数
private final int maxPendingTasks;

构造方法

protected SingleThreadEventExecutor(EventExecutorGroup parent, Executor executor,
                                        boolean addTaskWakesUp, int maxPendingTasks,
                                        RejectedExecutionHandler rejectedHandler) {
    super(parent);
    // 是否只有在调用addTask来向任务队列中添加任务时才唤醒executor线程
    this.addTaskWakesUp = addTaskWakesUp;
    // 最大pedning任务个数
    this.maxPendingTasks = Math.max(16, maxPendingTasks);
    this.executor = ObjectUtil.checkNotNull(executor, "executor");
    // 任务队列
    taskQueue = newTaskQueue(this.maxPendingTasks);
    rejectedExecutionHandler = ObjectUtil.checkNotNull(rejectedHandler, "rejectedHandler");
}

inEventLoop

判断指定的线程是否在当前eventLoop中

@Override
public boolean inEventLoop(Thread thread) {
    return thread == this.thread;
}

offerTask

final boolean offerTask(Runnable task) {
	// 判断当前状态是否已经关闭或者终结
    if (isShutdown()) {
    	// 抛出拒绝异常
        reject();
    }
    // 向任务队列中添加任务
    return taskQueue.offer(task);
}

addTask

protected void addTask(Runnable task) {
	// 判断待添加的任务是否是空，如果为空抛出异常
    if (task == null) {
        throw new NullPointerException("task");
    }
    // 向任务队列添加任务，如果失败调用reject
    if (!offerTask(task)) {
        reject(task);
    }
}

peekTask

查看当前任务队列中队头的任务，要求当前线程在eventLoop中

protected Runnable peekTask() {
    assert inEventLoop();
    return taskQueue.peek();
}

removeTask

从队列中移除任务

protected boolean removeTask(Runnable task) {
    if (task == null) {
        throw new NullPointerException("task");
    }
    return taskQueue.remove(task);
}

reject

当添加任务失败的时候，会调用reject方法

protected final void reject(Runnable task) {
   rejectedExecutionHandler.rejected(task, this);
}

RejectedExecutionHandlers这个类中提供了RejectedExecutorHandler的一些实现类
下面看下源码

public final class RejectedExecutionHandlers {
    private static final RejectedExecutionHandler REJECT = new RejectedExecutionHandler() {
        @Override
        public void rejected(Runnable task, SingleThreadEventExecutor executor) {
            throw new RejectedExecutionException();
        }
    };

    private RejectedExecutionHandlers() { }

    /**
     * Returns a {@link RejectedExecutionHandler} that will always just throw a {@link RejectedExecutionException}.
     */
    // 第一种实现，直接抛出异常
    public static RejectedExecutionHandler reject() {
        return REJECT;
    }

    /**
     * Tries to backoff when the task can not be added due restrictions for an configured amount of time. This
     * is only done if the task was added from outside of the event loop which means
     * {@link EventExecutor#inEventLoop()} returns {@code false}.
     */
     // 第二种实现，会尝试将任务添加到队列中，失败重试
    public static RejectedExecutionHandler backoff(final int retries, long backoffAmount, TimeUnit unit) {
        ObjectUtil.checkPositive(retries, "retries");
        final long backOffNanos = unit.toNanos(backoffAmount);
        return new RejectedExecutionHandler() {
            @Override
            public void rejected(Runnable task, SingleThreadEventExecutor executor) {
                if (!executor.inEventLoop()) {
                	// 循环尝试将任务添加到队列中
                    for (int i = 0; i < retries; i++) {
                    	// 尝试唤醒执行器，这样任务队列中的任务就会减少
                        // Try to wakup the executor so it will empty its task queue.
                        executor.wakeup(false);
						
						// 阻塞等待指定的时间
                        LockSupport.parkNanos(backOffNanos);
                        // 尝试将任务添加到队列中
                        if (executor.offerTask(task)) {
                            return;
                        }
                    }
                }
                // Either we tried to add the task from within the EventLoop or we was not able to add it even with
                // backoff.
                throw new RejectedExecutionException();
            }
        };
    }
}

execute

下面看下比较重要的方法execute，看是如何执行任务的

public void execute(Runnable task) {
    if (task == null) {
        throw new NullPointerException("task");
    }

	// 判断当前线程是否是eventLoop绑定的线程
    boolean inEventLoop = inEventLoop();
    // 将任务添加到任务队列中
    addTask(task);
    if (!inEventLoop) {
    	// 不在eventLoop绑定的线程，启动线程
        startThread();
        // 如果eventLoop已经关闭，并且移除任务失败，那么调用reject方法
        if (isShutdown() && removeTask(task)) {
            reject();
        }
    }

	// 如果addTaskWakesUp设置为false,那么会唤醒
	// addTaskWakesUp代表当向任务队列中添加任务时，是否会自动唤醒
	// 如果不会自动唤醒，则需要调用wakeup进行主动唤醒
	// wakesupForTask方法默认返回true，子类可以有自己的实现
    if (!addTaskWakesUp && wakesUpForTask(task)) {
    	// 唤醒
        wakeup(inEventLoop);
    }
}

startThread

private void startThread() {
	// 判断当前的状态是否是ST_NOT_STARTED
    if (state == ST_NOT_STARTED) {
    	// 将当前状态设置为ST_STARTED
        if (STATE_UPDATER.compareAndSet(this, ST_NOT_STARTED, ST_STARTED)) {
            try {
            	// 真正启动线程的逻辑
                doStartThread();
            } catch (Throwable cause) {
            	// 如果启动失败，将状态设置为ST_NOT_STARTED
                STATE_UPDATER.set(this, ST_NOT_STARTED);
                PlatformDependent.throwException(cause);
            }
        }
    }
}

private void doStartThread() {
	// 判断线程是否是空
    assert thread == null;
    // 将任务提交给executor进行执行
    executor.execute(new Runnable() {
        @Override
        public void run() {
        	// 将thread设置为执行当前任务的线程，这个就是每个SingleThreadEventExecutor独占线程的实现方式
            thread = Thread.currentThread();
            if (interrupted) {
                thread.interrupt();
            }

            boolean success = false;
            // 更新最近执行时间
            updateLastExecutionTime();
            try {
            	// 执行任务
                SingleThreadEventExecutor.this.run();
                success = true;
            } catch (Throwable t) {
                logger.warn("Unexpected exception from an event executor: ", t);
            } finally {
            	// 优雅关闭，将状态设置为ST_SHUTTING_DOWN
                for (;;) {
                    int oldState = state;
                    if (oldState >= ST_SHUTTING_DOWN || STATE_UPDATER.compareAndSet(
                            SingleThreadEventExecutor.this, oldState, ST_SHUTTING_DOWN)) {
                        break;
                    }
                }

                // Check if confirmShutdown() was called at the end of the loop.
                if (success && gracefulShutdownStartTime == 0) {
                    if (logger.isErrorEnabled()) {
                        logger.error("Buggy " + EventExecutor.class.getSimpleName() + " implementation; " +
                                SingleThreadEventExecutor.class.getSimpleName() + ".confirmShutdown() must " +
                                "be called before run() implementation terminates.");
                    }
                }

                try {
                	// 运行剩余的任务和关闭钩子
                    // Run all remaining tasks and shutdown hooks.
                    for (;;) {
                        if (confirmShutdown()) {
                            break;
                        }
                    }
                } finally {
                    try {
                    	// 清理资源
                        cleanup();
                    } finally {
                    	// 将状态设置为ST_TERMINATED
                        STATE_UPDATER.set(SingleThreadEventExecutor.this, ST_TERMINATED);
                        threadLock.release();
                        if (!taskQueue.isEmpty()) {
                            if (logger.isWarnEnabled()) {
                                logger.warn("An event executor terminated with " +
                                        "non-empty task queue (" + taskQueue.size() + ')');
                            }
                        }

                        terminationFuture.setSuccess(null);
                    }
                }
            }
        }
    });
}

run

SingleThreadEventExecutor中的run方法是一个抽象方法，由子类来实现

protected abstract void run();

invokeAll

invokeAll执行给定的多个任务，等待所有任务执行完毕

public <T> List<java.util.concurrent.Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
            throws InterruptedException {
    // 判断当前线程是否在EventLoop中
    throwIfInEventLoop("invokeAll");
    // 调用父类的invokeAll方法
    return super.invokeAll(tasks);
}

public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
        throws InterruptedException {
    if (tasks == null)
        throw new NullPointerException();
    ArrayList<Future<T>> futures = new ArrayList<Future<T>>(tasks.size());
    boolean done = false;
    try {
    	// 为每个任务创建一个RunableFuture，并且添加到futures列表中
    	// 然后执行该任务
        for (Callable<T> t : tasks) {
            RunnableFuture<T> f = newTaskFor(t);
            futures.add(f);
            execute(f);
        }
        // 阻塞等待所有任务执行完毕
        for (int i = 0, size = futures.size(); i < size; i++) {
            Future<T> f = futures.get(i);
            if (!f.isDone()) {
                try {
                    f.get();
                } catch (CancellationException ignore) {
                } catch (ExecutionException ignore) {
                }
            }
        }
        done = true;
        return futures;
    } finally {
        if (!done)
            for (int i = 0, size = futures.size(); i < size; i++)
                futures.get(i).cancel(true);
    }
}

invokeAny

给定多个任务，只要其中一个执行完毕就返回

@Override
public <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException {
	// 判断当前线程是否在eventloop中
    throwIfInEventLoop("invokeAny");
    // 调用父类的invokeAny方法
    return super.invokeAny(tasks);
}

public <T> T invokeAny(Collection<? extends Callable<T>> tasks)
        throws InterruptedException, ExecutionException {
    try {
        return doInvokeAny(tasks, false, 0);
    } catch (TimeoutException cannotHappen) {
        assert false;
        return null;
    }
}

private <T> T doInvokeAny(Collection<? extends Callable<T>> tasks,
                              boolean timed, long nanos)
        throws InterruptedException, ExecutionException, TimeoutException {
    if (tasks == null)
        throw new NullPointerException();
    int ntasks = tasks.size();
    if (ntasks == 0)
        throw new IllegalArgumentException();
    ArrayList<Future<T>> futures = new ArrayList<Future<T>>(ntasks);
    // 当向ExecutorCompletionService提交多个任务后
    // 调用take方法会返回已经完成的任务的future，如果当前没有已经完成的任务，那么会阻塞等待
    // 调用poll方法和take相似，只是如果当前没有已经完成的任务，会返回Null
    ExecutorCompletionService<T> ecs =
        new ExecutorCompletionService<T>(this);

    // For efficiency, especially in executors with limited
    // parallelism, check to see if previously submitted tasks are
    // done before submitting more of them. This interleaving
    // plus the exception mechanics account for messiness of main
    // loop.

    try {
        // Record exceptions so that if we fail to obtain any
        // result, we can throw the last exception we got.
        ExecutionException ee = null;
        final long deadline = timed ? System.nanoTime() + nanos : 0L;
        Iterator<? extends Callable<T>> it = tasks.iterator();

        // Start one task for sure; the rest incrementally
        // 渐进式地提交任务，并不是一次将所有的任务都提交
        futures.add(ecs.submit(it.next()));
        // 减少待提交任务数量
        --ntasks;
        // 增加当前正在运行的任务数量
        int active = 1;

        for (;;) {
        	// 取出当前已经运行成功的任务的future
            Future<T> f = ecs.poll();
           	// 为null代表当前没有完成的任务
            if (f == null) {
            	// 如果仍然有没有提交的任务，那么提交一个
                if (ntasks > 0) {
                	// 减少待提交任务数量
                    --ntasks;
                    // 提交任务
                    futures.add(ecs.submit(it.next()));
                    // 增加当前正在运行的任务数量
                    ++active;
                }
                // 提交的任务全部执行完成，退出循环
                else if (active == 0)
                    break;
                else if (timed) {
                    f = ecs.poll(nanos, TimeUnit.NANOSECONDS);
                    if (f == null)
                        throw new TimeoutException();
                    nanos = deadline - System.nanoTime();
                }
                else
                    f = ecs.take();
            }
            // 有执行成功的任务
            if (f != null) {
                --active;
                try {
                	// 返回执行结果
                    return f.get();
                } catch (ExecutionException eex) {
                    ee = eex;
                } catch (RuntimeException rex) {
                    ee = new ExecutionException(rex);
                }
            }
        }

        if (ee == null)
            ee = new ExecutionException();
        throw ee;

    } finally {
        for (int i = 0, size = futures.size(); i < size; i++)
            futures.get(i).cancel(true);
    }
}

wakeup

protected void wakeup(boolean inEventLoop) {
 // 如果在eventloop中，代表eventloop正在运行
 if (!inEventLoop || state == ST_SHUTTING_DOWN) {
        // Use offer as we actually only need this to unblock the thread and if offer fails we do not care as there
        // is already something in the queue.
        // 向任务队列中添加唤醒任务
        taskQueue.offer(WAKEUP_TASK);
    }
}

SingleThreadEventLoop

重要属性

// 默认最大pending任务个数
protected static final int DEFAULT_MAX_PENDING_TASKS = Math.max(16,
            SystemPropertyUtil.getInt("io.netty.eventLoop.maxPendingTasks", Integer.MAX_VALUE));
// 尾部任务队列，执行在taskQueue后面
private final Queue<Runnable> tailTasks;

register

@Override
public ChannelFuture register(Channel channel) {
    return register(new DefaultChannelPromise(channel, this));
}
public ChannelFuture register(final ChannelPromise promise) {
    ObjectUtil.checkNotNull(promise, "promise");
    // 将channel注册到eventloop上
    promise.channel().unsafe().register(this, promise);
    return promise;
}

hasTasks

判断是否有没有执行的任务
结合判断taskQueue和tailTask

protected boolean hasTasks() {
    return super.hasTasks() || !tailTasks.isEmpty();
}

pendingTasks

获取pending任务的个数

@Override
public int pendingTasks() {
    return super.pendingTasks() + tailTasks.size();
}

executeAfterEventLoopIteration

public final void executeAfterEventLoopIteration(Runnable task) {
    ObjectUtil.checkNotNull(task, "task");
    // 如果已经关闭，那么拒绝，并且执行拒绝回调
    if (isShutdown()) {
        reject();
    }

	// 向tailTask队列中添加任务
    if (!tailTasks.offer(task)) {
    	// 添加失败，拒绝任务
        reject(task);
    }
	
	// 唤醒
    if (wakesUpForTask(task)) {
        wakeup(inEventLoop());
    }
}

removeAfterEventLoopIterationTask

移除指定的任务

final boolean removeAfterEventLoopIterationTask(Runnable task) {
    return tailTasks.remove(ObjectUtil.checkNotNull(task, "task"));
}

afterRunningAllTasks

运行tailTask中的所有任务

protected void afterRunningAllTasks() {
    runAllTasksFrom(tailTasks);
}

NioEventLoop

重要属性

private static final int CLEANUP_INTERVAL = 256; // XXX Hard-coded value, but won't need customization.

// 是否禁用keyset优化
private static final boolean DISABLE_KEYSET_OPTIMIZATION =
        SystemPropertyUtil.getBoolean("io.netty.noKeySetOptimization", false);
// 空轮询创建新的selector最少的轮询次数
private static final int MIN_PREMATURE_SELECTOR_RETURNS = 3;
// 空轮询指定次数后，创建新的selector对象
private static final int SELECTOR_AUTO_REBUILD_THRESHOLD;

// Workaround for JDK NIO bug.
//
// See:
// - http://bugs.sun.com/view_bug.do?bug_id=6427854
// - https://github.com/netty/netty/issues/203
static {
    final String key = "sun.nio.ch.bugLevel";
    final String buglevel = SystemPropertyUtil.get(key);
    if (buglevel == null) {
        try {
            AccessController.doPrivileged(new PrivilegedAction<Void>() {
                @Override
                public Void run() {
                    System.setProperty(key, "");
                    return null;
                }
            });
        } catch (final SecurityException e) {
            logger.debug("Unable to get/set System Property: " + key, e);
        }
    }

    int selectorAutoRebuildThreshold = SystemPropertyUtil.getInt("io.netty.selectorAutoRebuildThreshold", 512);
    if (selectorAutoRebuildThreshold < MIN_PREMATURE_SELECTOR_RETURNS) {
        selectorAutoRebuildThreshold = 0;
    }

    SELECTOR_AUTO_REBUILD_THRESHOLD = selectorAutoRebuildThreshold;

    if (logger.isDebugEnabled()) {
        logger.debug("-Dio.netty.noKeySetOptimization: {}", DISABLE_KEYSET_OPTIMIZATION);
        logger.debug("-Dio.netty.selectorAutoRebuildThreshold: {}", SELECTOR_AUTO_REBUILD_THRESHOLD);
    }
}

/**
 * The NIO {@link Selector}.
 */
// 包装过的selector
private Selector selector;
// 未包装过的selector
private Selector unwrappedSelector;
// 注册的事件
private SelectedSelectionKeySet selectedKeys;
// 用于创建selector
private final SelectorProvider provider;

/**
 * Boolean that controls determines if a blocked Selector.select should
 * break out of its selection process. In our case we use a timeout for
 * the select method and the select method will block for that time unless
 * waken up.
 */
// 唤醒标记，用来决定selector.select是否需要停止
private final AtomicBoolean wakenUp = new AtomicBoolean();
// select策略
private final SelectStrategy selectStrategy;
// 处理io事件占比
private volatile int ioRatio = 50;
// 取消的selectionKey的数量
private int cancelledKeys;
// 是否需要再次select
private boolean needsToSelectAgain;

构造方法

NioEventLoop(NioEventLoopGroup parent, Executor executor, SelectorProvider selectorProvider,
                 SelectStrategy strategy, RejectedExecutionHandler rejectedExecutionHandler) {
    super(parent, executor, false, DEFAULT_MAX_PENDING_TASKS, rejectedExecutionHandler);
    if (selectorProvider == null) {
        throw new NullPointerException("selectorProvider");
    }
    if (strategy == null) {
        throw new NullPointerException("selectStrategy");
    }
    provider = selectorProvider;
    final SelectorTuple selectorTuple = openSelector();
    selector = selectorTuple.selector;
    unwrappedSelector = selectorTuple.unwrappedSelector;
    selectStrategy = strategy;
}

newTaskQueue

覆写父类版本
返回的是一个MpscQueue，是一个适用于多个生产线程和一个消费线程的队列

protected Queue<Runnable> newTaskQueue(int maxPendingTasks) {
        // This event loop never calls takeTask()
    return maxPendingTasks == Integer.MAX_VALUE ? PlatformDependent.<Runnable>newMpscQueue()
                                                : PlatformDependent.<Runnable>newMpscQueue(maxPendingTasks);
}
```
### pendingTasks
```java
public int pendingTasks() {
    // As we use a MpscQueue we need to ensure pendingTasks() is only executed from within the EventLoop as
    // otherwise we may see unexpected behavior (as size() is only allowed to be called by a single consumer).
    // See https://github.com/netty/netty/issues/5297
    // 因为使用的线程是多个生成线程，单个消费线程的队列，因此查看队列长度只允许在消费线程中进行
    if (inEventLoop()) {
        return super.pendingTasks();
    } else {
    	// 不在eventLoop中，即不在消费线程中，需要通过提交任务的方式来实现
    	// 提交的任务会在消费线程中执行
        return submit(pendingTasksCallable).syncUninterruptibly().getNow();
    }
}
```