容器CNI完全解读bridge实现（二）

之前介绍CNI基本操作，现在介绍一个bridge的实现。
它也实现了创建和删除接口。
先看创建接口：

func cmdAdd(args *skel.CmdArgs) error {
    n, cniVersion, err := loadNetConf(args.StdinData)
    if err != nil {
        return err
    }

    if n.IsDefaultGW {
        n.IsGW = true
    }

    br, brInterface, err := setupBridge(n)
    if err != nil {
        return err
    }

    netns, err := ns.GetNS(args.Netns)
    if err != nil {
        return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
    }
    defer netns.Close()

    hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
    if err != nil {
        return err
    }

    // run the IPAM plugin and get back the config to apply
    r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
    if err != nil {
        return err
    }

    // Convert whatever the IPAM result was into the current Result type
    result, err := current.NewResultFromResult(r)
    if err != nil {
        return err
    }

    if len(result.IPs) == 0 {
        return errors.New("IPAM plugin returned missing IP config")
    }

    result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}

    for _, ipc := range result.IPs {
        // All IPs currently refer to the container interface
        ipc.Interface = 2
        if ipc.Gateway == nil && n.IsGW {
            ipc.Gateway = calcGatewayIP(&ipc.Address)
        }
    }

    if err := netns.Do(func(_ ns.NetNS) error {
        // set the default gateway if requested
        if n.IsDefaultGW {
            for _, ipc := range result.IPs {
                defaultNet := &net.IPNet{}
                switch {
                case ipc.Address.IP.To4() != nil:
                    defaultNet.IP = net.IPv4zero
                    defaultNet.Mask = net.IPMask(net.IPv4zero)
                case len(ipc.Address.IP) == net.IPv6len && ipc.Address.IP.To4() == nil:
                    defaultNet.IP = net.IPv6zero
                    defaultNet.Mask = net.IPMask(net.IPv6zero)
                default:
                    return fmt.Errorf("Unknown IP object: %v", ipc)
                }

                for _, route := range result.Routes {
                    if defaultNet.String() == route.Dst.String() {
                        if route.GW != nil && !route.GW.Equal(ipc.Gateway) {
                            return fmt.Errorf(
                                "isDefaultGateway ineffective because IPAM sets default route via %q",
                                route.GW,
                            )
                        }
                    }
                }

                result.Routes = append(
                    result.Routes,
                    &types.Route{Dst: *defaultNet, GW: ipc.Gateway},
                )
            }
        }

        if err := ipam.ConfigureIface(args.IfName, result); err != nil {
            return err
        }

        if err := ip.SetHWAddrByIP(args.IfName, result.IPs[0].Address.IP, nil /* TODO IPv6 */); err != nil {
            return err
        }

        // Refetch the veth since its MAC address may changed
        link, err := netlink.LinkByName(args.IfName)
        if err != nil {
            return fmt.Errorf("could not lookup %q: %v", args.IfName, err)
        }
        containerInterface.Mac = link.Attrs().HardwareAddr.String()

        return nil
    }); err != nil {
        return err
    }

    if n.IsGW {
        var firstV4Addr net.IP
        for _, ipc := range result.IPs {
            gwn := &net.IPNet{
                IP:   ipc.Gateway,
                Mask: ipc.Address.Mask,
            }
            if ipc.Gateway.To4() != nil && firstV4Addr == nil {
                firstV4Addr = ipc.Gateway
            }

            if err = ensureBridgeAddr(br, gwn, n.ForceAddress); err != nil {
                return err
            }
        }

        if firstV4Addr != nil {
            if err := ip.SetHWAddrByIP(n.BrName, firstV4Addr, nil /* TODO IPv6 */); err != nil {
                return err
            }
        }

        if err := ip.EnableIP4Forward(); err != nil {
            return fmt.Errorf("failed to enable forwarding: %v", err)
        }
    }

    if n.IPMasq {
        chain := utils.FormatChainName(n.Name, args.ContainerID)
        comment := utils.FormatComment(n.Name, args.ContainerID)
        for _, ipc := range result.IPs {
            if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
                return err
            }
        }
    }

    // Refetch the bridge since its MAC address may change when the first
    // veth is added or after its IP address is set
    br, err = bridgeByName(n.BrName)
    if err != nil {
        return err
    }
    brInterface.Mac = br.Attrs().HardwareAddr.String()

    result.DNS = n.DNS

    return types.PrintResult(result, cniVersion)
}

这个里面有几个步骤
1、创建网桥并启动网桥，
setupBridge里面调用ensureBridge，先通过err := netlink.LinkAdd(br)创建网桥，然后通过 err := netlink.LinkSetUp(br)启动网桥

2.创建veth，这个是一个管道，Linux的网卡对。
hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)
当前先通过hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)创建网卡对，一个是主机网卡一个容器网卡，并且把主机网卡设置成hairpin模式

3.通过ipam获取IP，这个是调用另一个获取IP的二进制文件ipam，这个里面返回一个网络配置列表（result.IPs）

4.配置容器内网卡，通过ipam.ConfigureIface和ip.SetHWAddrByIP配置容器的IP地址和网卡mac，其中SetHWAddrByIP通过IP地址计算出mac地址，详见
hwAddr, err := hwaddr.GenerateHardwareAddr4(ip4, hwaddr.PrivateMACPrefix)

5.配置网桥，这个里面配置网桥的IP地址和mac。最后如果可以设置ipmasq，这样容器就可以通过SNAT去连接外部网络了，这样容器就可以加入网络了

介绍完创建的过程，删除的过程就简单了。

func cmdDel(args *skel.CmdArgs) error {
    n, _, err := loadNetConf(args.StdinData)
    if err != nil {
        return err
    }

    if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
        return err
    }

    if args.Netns == "" {
        return nil
    }

    var ipn *net.IPNet
    err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
        var err error
        ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
        return err
    })
    if err != nil {
        return err
    }

    if n.IPMasq {
        chain := utils.FormatChainName(n.Name, args.ContainerID)
        comment := utils.FormatComment(n.Name, args.ContainerID)
        if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
            return err
        }
    }

    return nil
}

这个里面显示通过ipam.ExecDel是释放IP地址的占用，然后通过 ip.DelLinkByNameAddr去删除veth，如果设置了ipmasq，这是就可以通过TeardownIPMasq删除这些iptables规则。