etcd+flanneld不同节点分配到相同网段原因剖析

背景
最近同事在部署k8s集群时，不同的节点分配到的网段完全相同
etcd+flanneld不同节点分配到相同网段

1. 环境信息：

• etcd 3.2.12
• flannel v0.11
• OS Centos 7.6

2. flanneld为节点分配网段策略

从subnet代码库里面可以看到，当前flannel支持从etcd和kube api存储网段信息，这里在etcd里面存储，因此只分析在etcd作为网段存储时，地址分配流程

• 首先flannel启动时候，会获取本机的网络接口，iface或者regex未指定的话则会返回第一个接口，根据这个接口的IP信息来分配网段，首先从源码来看下具体的流程

	flannelFlags.Var(&opts.iface, "iface", "interface to use (IP or name) for inter-host communication. Can be specified multiple times to check each option in order. Returns the first match found.")

# 第一步，获取本机接口信息
# 如果 iface 和 iface-regex都未指定,则会返回LookupExtIface函数获取到的第一个接口
if len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 {
		extIface, err = LookupExtIface("", "")
		if err != nil {
			log.Error("Failed to find any valid interface to use: ", err)
			os.Exit(1)
		}
	} 
# LookupExtIface函数定义
func LookupExtIface(ifname string, ifregex string) (*backend.ExternalInterface, error) {
	var iface *net.Interface
	var ifaceAddr net.IP
	var err error

	if len(ifname) > 0 {
		if ifaceAddr = net.ParseIP(ifname); ifaceAddr != nil {
			log.Infof("Searching for interface using %s", ifaceAddr)
			iface, err = ip.GetInterfaceByIP(ifaceAddr)
			if err != nil {
				return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
			}
		} else {
			iface, err = net.InterfaceByName(ifname)
			if err != nil {
				return nil, fmt.Errorf("error looking up interface %s: %s", ifname, err)
			}
		}
	} else if len(ifregex) > 0 {
		// Use the regex if specified and the iface option for matching a specific ip or name is not used
		ifaces, err := net.Interfaces()
		if err != nil {
			return nil, fmt.Errorf("error listing all interfaces: %s", err)
		}

		// Check IP
		for _, ifaceToMatch := range ifaces {
			ifaceIP, err := ip.GetIfaceIP4Addr(&ifaceToMatch)
			if err != nil {
				// Skip if there is no IPv4 address
				continue
			}

			matched, err := regexp.MatchString(ifregex, ifaceIP.String())
			if err != nil {
				return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
			}

			if matched {
				ifaceAddr = ifaceIP
				iface = &ifaceToMatch
				break
			}
		}

		// Check Name
		if iface == nil && ifaceAddr == nil {
			for _, ifaceToMatch := range ifaces {
				matched, err := regexp.MatchString(ifregex, ifaceToMatch.Name)
				if err != nil {
					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceToMatch.Name)
				}

				if matched {
					iface = &ifaceToMatch
					break
				}
			}
		}

		// Check that nothing was matched
		if iface == nil {
			var availableFaces []string
			for _, f := range ifaces {
				ip, _ := ip.GetIfaceIP4Addr(&f) // We can safely ignore errors. We just won't log any ip
				availableFaces = append(availableFaces, fmt.Sprintf("%s:%s", f.Name, ip))
			}

			return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregex, strings.Join(availableFaces, ", "))
		}
	} else {
		log.Info("Determining IP address of default interface")
		if iface, err = ip.GetDefaultGatewayIface(); err != nil {
			return nil, fmt.Errorf("failed to get default interface: %s", err)
		}
	}

	if ifaceAddr == nil {
		ifaceAddr, err = ip.GetIfaceIP4Addr(iface)
		if err != nil {
			return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
		}
	}

	log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)

	if iface.MTU == 0 {
		return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
	}

	var extAddr net.IP

	if len(opts.publicIP) > 0 {
		extAddr = net.ParseIP(opts.publicIP)
		if extAddr == nil {
			return nil, fmt.Errorf("invalid public IP address: %s", opts.publicIP)
		}
		log.Infof("Using %s as external address", extAddr)
	}
	if extAddr == nil {
		log.Infof("Defaulting external address to interface address (%s)", ifaceAddr)
		extAddr = ifaceAddr
	}
# 上述条件都不符合的话会返回下面的值
	return &backend.ExternalInterface{
		Iface:     iface,
		IfaceAddr: ifaceAddr,
		ExtAddr:   extAddr,
	}, nil
}
# 看下ExternalInterface定义（本机全部网络接口的值）
type ExternalInterface struct {
        Iface     *net.Interface 
        IfaceAddr net.IP
        ExtAddr   net.IP
}

# 第二步，分配subnet
func (esr *etcdSubnetRegistry) createSubnet(ctx context.Context, sn ip.IP4Net, attrs *LeaseAttrs, ttl time.Duration) (time.Time, error) {
	key := path.Join(esr.etcdCfg.Prefix, "subnets", MakeSubnetKey(sn))
	value, err := json.Marshal(attrs)
	if err != nil {
		return time.Time{}, err
	}
# 可以看到会从etcd指定路径下获取subnets的信息,调用MakeSubnetKey函数如下：
  func MakeSubnetKey(sn ip.IP4Net) string {
	return sn.StringSep(".", "-")
}
# 此步会调用StringSep函数，如下：
func (ip IP4) StringSep(sep string) string {
	a, b, c, d := ip.Octets()
	return fmt.Sprintf("%d%s%d%s%d%s%d", a, sep, b, sep, c, sep, d)
}
# 会根据第一个字段ip去重，这个就是为什么不同节点会分配到相同的网段信息

原因

环境为vbox虚拟机，vbox虚拟机默认的eth0地址10.0.2.15，不同的vbox虚拟机都会有这个地址，而iface又未特别指定，所以会出现不同节点分配到不同IP的情况，

既然原因已经查明，解决办法就有了，步骤如下：

1. 修改flanneld启动文件参数，指定iface

ExecStart=/usr/local/bin/flanneld -iface=eth1 \
  -etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  -etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
  -etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
  -etcd-endpoints=https://10.0.0.111:2379,https://10.0.0.112:2379,https://10.0.0.113:2379 \
  -etcd-prefix=/kubernetes/network

2. 删除flannel已经生成的网络配置文件

rm -rf /run/flannel/*

3. reload和重启flannel服务

# 使用systemd管理的服务执行如下，不用systemd的可以按自己的方式来
systemctl daemon-reload 
systemctl restart flanneld.service

4. 查看验证

# 命令行查看
etcdctl --endpoints=https://10.0.0.111:2379,https://10.0.0.112:2379,https://10.0.0.113:2379 --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem ls  /kubernetes/network/subnets
# 结果如下
/kubernetes/network/subnets/10.1.100.0-24
/kubernetes/network/subnets/10.1.26.0-24
/kubernetes/network/subnets/10.1.11.0-24
/kubernetes/network/subnets/10.1.77.0-24