Android 10 路由添加原理

以 WiFi 连接网络的过程为例分析路由表添加原理
WiFi 工作在 client 模式时会调用 ClientModeImpl 状态机，进入 ConnectModeState 状态，从而调用 setupClientMode 函数：
[ClientModeImpl.java]

private void setupClientMode() {
    ...
    updateDataInterface();
    ...
}

updateDataInterface 函数的实现：

private void updateDataInterface() {
    ...
    mIpClientCallbacks = new IpClientCallbacksImpl();    // 创建回调，由服务端调用
    mFacade.makeIpClient(mContext, mDataInterfaceName, mIpClientCallbacks);    // 通过 netstack 服务创建 ipclient
    if (!mIpClientCallbacks.awaitCreation()) {
        loge("Timeout waiting for IpClient");
    }
    ...
}

makeIpClient 会调用 netstack 服务的 makeIpClient 函数：

@Override
public void makeIpClient(String ifName, IIpClientCallbacks cb) throws RemoteException {
    checkNetworkStackCallingPermission();
    updateSystemAidlVersion(cb.getInterfaceVersion());
    final IpClient ipClient = new IpClient(mContext, ifName, cb, mObserverRegistry, this);

    synchronized (mIpClients) {
        final Iterator<WeakReference<IpClient>> it = mIpClients.iterator();
        while (it.hasNext()) {
            final IpClient ipc = it.next().get();
            if (ipc == null) {
                it.remove();
            }
        }
        mIpClients.add(new WeakReference<>(ipClient));
    }

    cb.onIpClientCreated(ipClient.makeConnector());
}

这里值得注意的是 mObserverRegistry 在哪里初始化？答案是在 NetworkStackConnector 的构造函数中：

NetworkStackConnector(Context context) {
    mContext = context;
    mNetd = INetd.Stub.asInterface(
            (IBinder) context.getSystemService(Context.NETD_SERVICE));    // 获取 netd 服务
    mObserverRegistry = new NetworkObserverRegistry();
    mCm = context.getSystemService(ConnectivityManager.class);
    mIpMemoryStoreService = new IpMemoryStoreService(context);

    int netdVersion;
    try {
        netdVersion = mNetd.getInterfaceVersion();
    } catch (RemoteException e) {
        mLog.e("Error obtaining INetd version", e);
        netdVersion = -1;
    }
    mNetdAidlVersion = netdVersion;

    try {
        mObserverRegistry.register(mNetd);
    } catch (RemoteException e) {
        mLog.e("Error registering observer on Netd", e);
    }
}

NetworkStackConnector 是 netstack 服务所有实现。上面的构造函数中，先获取了 netd 服务，然后初始化了mObserverRegistry，紧接着调用其 register 函数：

public class NetworkObserverRegistry extends INetdUnsolicitedEventListener.Stub {
    ...
    void register(@NonNull INetd netd) throws RemoteException {
        netd.registerUnsolicitedEventListener(this);
    }
    ...
}

register 函数向 netd 服务注册事件回调，而 mObserverRegistry 本身就是回调对象，通过 NetworkObserverRegistry 类的继承关系可见一斑。那么看一下 netd 中注册事件回调函数：

binder::Status NetdNativeService::registerUnsolicitedEventListener(
        const android::sp<android::net::INetdUnsolicitedEventListener>& listener) {
    ENFORCE_NETWORK_STACK_PERMISSIONS();
    gCtls->eventReporter.registerUnsolEventListener(listener);
    return binder::Status::ok();
}

在 EventReporter 类中找到 registerUnsolEventListener 成员函数的定义：

void EventReporter::registerUnsolEventListener(
        const android::sp<INetdUnsolicitedEventListener>& listener) {
    ...
    mUnsolListenerMap.insert({listener, deathRecipient});
}

这里的注册仅将 listener 添加到 mUnsolListenerMap 容器中，通过 getNetdUnsolicitedEventListenerMap 可以获取已注册的 listener：

EventReporter::UnsolListenerMap EventReporter::getNetdUnsolicitedEventListenerMap() const {
    std::lock_guard lock(mUnsolicitedMutex);
    return mUnsolListenerMap;
}

具体的 listener 在哪里使用呢？这必须先看 NetlinkManager 的实现，在 NetlinkManager中通过 netlink 与 kernel 进行交互，并且通过 NetlinkHandler 处理 kernel 上报的 event。

NetlinkHandler *NetlinkManager::setupSocket(int *sock, int netlinkFamily,
    int groups, int format, bool configNflog) {
    ...
    nladdr.nl_family = AF_NETLINK;
    // Kernel will assign a unique nl_pid if set to zero.
    nladdr.nl_pid = 0;    // 与 kernel 通信
    nladdr.nl_groups = groups;

    if ((*sock = socket(PF_NETLINK, SOCK_DGRAM | SOCK_CLOEXEC, netlinkFamily)) < 0) {    // 创建 netlink 套接字
        ALOGE("Unable to create netlink socket for family %d: %s", netlinkFamily, strerror(errno));
        return nullptr;
    }
    ...
    NetlinkHandler *handler = new NetlinkHandler(this, *sock, format);    // 创建 NetlinkHandler 接收消息
    if (handler->start()) {
        ALOGE("Unable to start NetlinkHandler: %s", strerror(errno));
        close(*sock);
        return nullptr;
    }
    ...
}

我们关心的是 NetlinkHandler 处理消息的部分：

class NetlinkHandler : public ::NetlinkListener {
  ...
  protected:
    virtual void onEvent(NetlinkEvent *evt);
}

首先 NetlinkHandler 继承至 NetlinkListener，当 NetlinkListener 监听的套接字收到 kernel 传来的消息时，会调用子类的 onEvent 处理消息：

void NetlinkHandler::onEvent(NetlinkEvent *evt) {
    const char *subsys = evt->getSubsystem();
    if (!subsys) {
        ALOGW("No subsystem found in netlink event");
        return;
    }

    if (!strcmp(subsys, "net")) {    // 判断为网络子系统
        NetlinkEvent::Action action = evt->getAction();
        const char *iface = evt->findParam("INTERFACE");
        ...
        if (action == NetlinkEvent::Action::kRouteUpdated ||
                   action == NetlinkEvent::Action::kRouteRemoved) {
            const char *route = evt->findParam("ROUTE");
            const char *gateway = evt->findParam("GATEWAY");
            const char *iface = evt->findParam("INTERFACE");
            if (route && (gateway || iface)) {
                notifyRouteChange((action == NetlinkEvent::Action::kRouteUpdated) ? true : false,
                                  route, (gateway == nullptr) ? "" : gateway,
                                  (iface == nullptr) ? "" : iface);
            }
        }
        ...
    }
}

但路由发生变化时，kernel 会上报 kRouteUpdated 事件消息，这时会调用 notifyRouteChange 处理消息：

void NetlinkHandler::notifyRouteChange(bool updated, const std::string& route,
                                       const std::string& gateway, const std::string& ifName) {
    LOG_EVENT_FUNC(BINDER_RETRY, onRouteChanged, updated, route, gateway, ifName);
}

LOG_EVENT_FUNC 宏定义如下：

#define LOG_EVENT_FUNC(retry, func, ...)                                                    \
    do {                                                                                    \
        const auto listenerMap = gCtls->eventReporter.getNetdUnsolicitedEventListenerMap(); \
        for (auto& listener : listenerMap) {                                                \
            auto entry = gUnsolicitedLog.newEntry().function(#func).args(__VA_ARGS__);      \
            if (retry(listener.first->func(__VA_ARGS__))) {                                 \
                gUnsolicitedLog.log(entry.withAutomaticDuration());                         \
            }                                                                               \
        }                                                                                   \
    } while (0)

这里会遍历之前注册的 listener，并调用 listen 的 func 函数，宏展开后对应的会调用 onRouteChanged 函数。此时会回调 mObserverRegistry 的 onRouteChanged 函数：

public void onRouteChanged(boolean updated, String route, String gateway, String ifName) {
    final RouteInfo processRoute = new RouteInfo(new IpPrefix(route),
            ("".equals(gateway)) ? null : InetAddresses.parseNumericAddress(gateway),
            ifName, RTN_UNICAST);
    if (updated) {
        invokeForAllObservers(o -> o.onRouteUpdated(processRoute));
    } else {
        invokeForAllObservers(o -> o.onRouteRemoved(processRoute));
    }
}

invokeForAllObservers 会遍历之前注册过的 observer，并调用 observer 的 onRouteUpdated 函数。
observer 又是在哪里注册呢？这又得回到 makeIpClient 函数创建 IpClient 对象说起，在 IpClient 的构造函数中注册了 mLinkObserver：

IpClient(Context context, String ifName, IIpClientCallbacks callback,
        NetworkObserverRegistry observerRegistry, NetworkStackServiceManager nssManager,
        Dependencies deps) {
    ...
    mLinkObserver = new IpClientLinkObserver(
            mInterfaceName,
            () -> sendMessage(EVENT_NETLINK_LINKPROPERTIES_CHANGED)) {
        @Override
        public void onInterfaceAdded(String iface) {
            super.onInterfaceAdded(iface);
            ...
        }


        @Override
        public void onInterfaceRemoved(String iface) {
            super.onInterfaceRemoved(iface);
            ...
        }
    };
    ...
    startStateMachineUpdaters();
}

startStateMachineUpdaters 会注册前面定义的 mLinkObserver：

private void startStateMachineUpdaters() {
    mObserverRegistry.registerObserverForNonblockingCallback(mLinkObserver);
}

继续分析 kRouteUpdated 消息的传递，经过前面分析 mLinkObserver 的 onRouteUpdated 函数会被调用：

public void onRouteUpdated(RouteInfo route) {
    if (mInterfaceName.equals(route.getInterface())) {
        maybeLog("routeUpdated", route);
        boolean changed;
        synchronized (this) {
            changed = mLinkProperties.addRoute(route);
        }
        if (changed) {
            mCallback.update();
        }
    }
}

这里的 mCallback 为 IpClientLinkObserver 构造函数中传递 lamda 表达式，因此会调用如下语句：

sendMessage(EVENT_NETLINK_LINKPROPERTIES_CHANGED)

在 IpClient 中处理消息：

case EVENT_NETLINK_LINKPROPERTIES_CHANGED:
    handleLinkPropertiesUpdate(NO_CALLBACKS);
    break;

handleLinkPropertiesUpdate 中会分发消息：

dispatchCallback(delta, newLp);

dispatchCallback 函数中默认的消息处理如下：

mCallback.onLinkPropertiesChange(newLp);

mCallback 为 IpClient 的成员变量，其值为 ClientModeImpl 出过来的 mIpClientCallbacks 对象的包装。因此 IpClientCallbacksImpl 的 onLinkPropertiesChange 会被调用：

public void onLinkPropertiesChange(LinkProperties newLp) {
    sendMessage(CMD_UPDATE_LINKPROPERTIES, newLp);
}

此消息通过 mNetworkAgent.sendLinkProperties(mLinkProperties) 传给 ConnectivityService 处理：

case NetworkAgent.EVENT_NETWORK_PROPERTIES_CHANGED: {
    handleUpdateLinkProperties(nai, (LinkProperties) msg.obj);
    break;
}

handleUpdateLinkProperties 会调用 updateLinkProperties 最终处理消息：

private void updateLinkProperties(NetworkAgentInfo networkAgent, LinkProperties newLp,
            LinkProperties oldLp) {
    ...
    updateRoutes(newLp, oldLp, netId);	// 更新路由表
    ...
}

小结：

• 注册 netlink 消息过程：
  ClientModeImpl -> netstack -> netd -> kernel
• 路由添加事件传递过程：
  kernel -> netd -> NetworkObserverRegistry -> IpClientCallbacksImpl

updateRoutes 接下来的工作就创建路由表并添加路由（只考虑添加的情况）。

private boolean updateRoutes(LinkProperties newLp, LinkProperties oldLp, int netId) {
    // Compare the route diff to determine which routes should be added and removed.
    CompareResult<RouteInfo> routeDiff = new CompareResult<RouteInfo>(
            oldLp != null ? oldLp.getAllRoutes() : null,
            newLp != null ? newLp.getAllRoutes() : null);
    for (RouteInfo route : routeDiff.added) {
        if (route.hasGateway()) continue;
        if (VDBG || DDBG) log("Adding Route [" + route + "] to network " + netId);
        try {
            mNMS.addRoute(netId, route);    // 添加路由
        } catch (Exception e) {
            if ((route.getDestination().getAddress() instanceof Inet4Address) || VDBG) {
                loge("Exception in addRoute for non-gateway: " + e);
            }
        }
    }
    for (RouteInfo route : routeDiff.added) {
        if (route.hasGateway() == false) continue;
        if (VDBG || DDBG) log("Adding Route [" + route + "] to network " + netId);
        try {
            mNMS.addRoute(netId, route);    // 删除路由
        } catch (Exception e) {
            if ((route.getGateway() instanceof Inet4Address) || VDBG) {
                loge("Exception in addRoute for gateway: " + e);
            }
        }
    }

    for (RouteInfo route : routeDiff.removed) {
        if (VDBG || DDBG) log("Removing Route [" + route + "] from network " + netId);
        try {
            mNMS.removeRoute(netId, route);
        } catch (Exception e) {
            loge("Exception in removeRoute: " + e);
        }
    }
    return !routeDiff.added.isEmpty() || !routeDiff.removed.isEmpty();
}

mNMS 为 INetworkManagementService 代理端对象，服务端实现了 addRoute 函数：
[NetworkManagementService.java]

public void addRoute(int netId, RouteInfo route) {
    modifyRoute(MODIFY_OPERATION_ADD, netId, route);
}

modifyRoute 函数的实现：

private void modifyRoute(boolean add, int netId, RouteInfo route) {
    ...
        if (add) {
            mNetdService.networkAddRoute(netId, ifName, dst, nextHop);
        } else {
            mNetdService.networkRemoveRoute(netId, ifName, dst, nextHop);
        }
    ...
}

mNetdService 为 netd 服务的代理，那么会调用服务端的 networkAddRoute 函数：
[NetdNativeService.cpp]

binder::Status NetdNativeService::networkAddRoute(int32_t netId, const std::string& ifName,
                                                  const std::string& destination,
                                                  const std::string& nextHop) {
    ...
    int res = gCtls->netCtrl.addRoute(netId, ifName.c_str(), destination.c_str(),
                                      nextHop.empty() ? nullptr : nextHop.c_str(), legacy, uid);
    return statusFromErrcode(res);
}

NetworkController 中的 addRoute 实现：
[NetworkController.cpp]

int NetworkController::addRoute(unsigned netId, const char* interface, const char* destination,
                                const char* nexthop, bool legacy, uid_t uid) {
    return modifyRoute(netId, interface, destination, nexthop, true, legacy, uid);
}

NetworkController 中 modifyRoute 源码：

int NetworkController::modifyRoute(unsigned netId, const char* interface, const char* destination,
                                   const char* nexthop, bool add, bool legacy, uid_t uid) {
    ...
    return add ? RouteController::addRoute(interface, destination, nexthop, tableType) :
                 RouteController::removeRoute(interface, destination, nexthop, tableType);
}

RouteController 中的 addRoute 实现：
[RouteController.cpp]

int RouteController::addRoute(const char* interface, const char* destination, const char* nexthop,
                              TableType tableType) {
    return modifyRoute(RTM_NEWROUTE, interface, destination, nexthop, tableType);
}

RouteController 中的 modifyRoute 实现：

WARN_UNUSED_RESULT int RouteController::modifyRoute(uint16_t action, const char* interface,
                                                    const char* destination, const char* nexthop,
                                                    TableType tableType) {
    uint32_t table;
    switch (tableType) {    // 路由表的类型
        case RouteController::INTERFACE: {    // 每一个网卡对应一张路由表
            table = getRouteTableForInterface(interface);    // 通过网卡名找到对应的路由表
            if (table == RT_TABLE_UNSPEC) {
                return -ESRCH;
            }
            break;
        }
        case RouteController::LOCAL_NETWORK: {
            table = ROUTE_TABLE_LOCAL_NETWORK;
            break;
        }
        case RouteController::LEGACY_NETWORK: {
            table = ROUTE_TABLE_LEGACY_NETWORK;
            break;
        }
        case RouteController::LEGACY_SYSTEM: {
            table = ROUTE_TABLE_LEGACY_SYSTEM;
            break;
        }
    }

    int ret = modifyIpRoute(action, table, interface, destination, nexthop);    // 添加路由信息
    // Trying to add a route that already exists shouldn't cause an error.
    if (ret && !(action == RTM_NEWROUTE && ret == -EEXIST)) {
        return ret;
    }

    return 0;
}

modifyIpRoute 函数是真正添加路由的地方，它通过 netlink 将路由添加到 kernel，具体代码这里不再分析。

WARN_UNUSED_RESULT int modifyIpRoute(uint16_t action, uint32_t table, const char* interface,
                                     const char* destination, const char* nexthop) {
    // At least the destination must be non-null.
    if (!destination) {
        ALOGE("null destination");
        return -EFAULT;
    }

    // Parse the prefix.
    uint8_t rawAddress[sizeof(in6_addr)];
    uint8_t family;
    uint8_t prefixLength;
    int rawLength = parsePrefix(destination, &family, rawAddress, sizeof(rawAddress),
                                &prefixLength);
    if (rawLength < 0) {
        ALOGE("parsePrefix failed for destination %s (%s)", destination, strerror(-rawLength));
        return rawLength;
    }

    if (static_cast<size_t>(rawLength) > sizeof(rawAddress)) {
        ALOGE("impossible! address too long (%d vs %zu)", rawLength, sizeof(rawAddress));
        return -ENOBUFS;  // Cannot happen; parsePrefix only supports IPv4 and IPv6.
    }

    uint8_t type = RTN_UNICAST;
    uint32_t ifindex;
    uint8_t rawNexthop[sizeof(in6_addr)];

    if (nexthop && !strcmp(nexthop, "unreachable")) {
        type = RTN_UNREACHABLE;
        // 'interface' is likely non-NULL, as the caller (modifyRoute()) likely used it to lookup
        // the table number. But it's an error to specify an interface ("dev ...") or a nexthop for
        // unreachable routes, so nuke them. (IPv6 allows them to be specified; IPv4 doesn't.)
        interface = OIF_NONE;
        nexthop = nullptr;
    } else if (nexthop && !strcmp(nexthop, "throw")) {
        type = RTN_THROW;
        interface = OIF_NONE;
        nexthop = nullptr;
    } else {
        // If an interface was specified, find the ifindex.
        if (interface != OIF_NONE) {
            ifindex = if_nametoindex(interface);
            if (!ifindex) {
                ALOGE("cannot find interface %s", interface);
                return -ENODEV;
            }
        }

        // If a nexthop was specified, parse it as the same family as the prefix.
        if (nexthop && inet_pton(family, nexthop, rawNexthop) <= 0) {
            ALOGE("inet_pton failed for nexthop %s", nexthop);
            return -EINVAL;
        }
    }

    bool isDefaultThrowRoute = (type == RTN_THROW && prefixLength == 0);

    // Assemble a rtmsg and put it in an array of iovec structures.
    rtmsg route = {
        .rtm_protocol = RTPROT_STATIC,
        .rtm_type = type,
        .rtm_family = family,
        .rtm_dst_len = prefixLength,
        .rtm_scope = static_cast<uint8_t>(nexthop ? RT_SCOPE_UNIVERSE : RT_SCOPE_LINK),
    };

    rtattr rtaDst     = { U16_RTA_LENGTH(rawLength), RTA_DST };
    rtattr rtaGateway = { U16_RTA_LENGTH(rawLength), RTA_GATEWAY };

    iovec iov[] = {
        { nullptr,          0 },
        { &route,        sizeof(route) },
        { &RTATTR_TABLE, sizeof(RTATTR_TABLE) },
        { &table,        sizeof(table) },
        { &rtaDst,       sizeof(rtaDst) },
        { rawAddress,    static_cast<size_t>(rawLength) },
        { &RTATTR_OIF,   interface != OIF_NONE ? sizeof(RTATTR_OIF) : 0 },
        { &ifindex,      interface != OIF_NONE ? sizeof(ifindex) : 0 },
        { &rtaGateway,   nexthop ? sizeof(rtaGateway) : 0 },
        { rawNexthop,    nexthop ? static_cast<size_t>(rawLength) : 0 },
        { &RTATTR_PRIO,  isDefaultThrowRoute ? sizeof(RTATTR_PRIO) : 0 },
        { &PRIO_THROW,   isDefaultThrowRoute ? sizeof(PRIO_THROW) : 0 },
    };

    uint16_t flags = (action == RTM_NEWROUTE) ? NETLINK_ROUTE_CREATE_FLAGS : NETLINK_REQUEST_FLAGS;

    // Allow creating multiple link-local routes in the same table, so we can make IPv6
    // work on all interfaces in the local_network table.
    if (family == AF_INET6 && IN6_IS_ADDR_LINKLOCAL(reinterpret_cast<in6_addr*>(rawAddress))) {
        flags &= ~NLM_F_EXCL;
    }

    int ret = sendNetlinkRequest(action, flags, iov, ARRAY_SIZE(iov), nullptr);
    if (ret) {
        ALOGE("Error %s route %s -> %s %s to table %u: %s",
              actionName(action), destination, nexthop, interface, table, strerror(-ret));
    }
    return ret;
}

至此已分析完路由添加的全过程。

疑问：
路由是 kernel 自动添加，然后通过 netlink 事件上报给上层，为何上层又要执行添加路由的动作，这不是重复了吗？
这是因为 linux 使用了策略路由，可以存在多张路由表，kernel 自动添加的路由会添加到主路由表中。上层添加的路由会创建自己的表。网络通讯时，通过策略选择适合的路由表。