kubernetes之kube-proxy ipvs userspace iptables 三种模式源码分析

kubernetes 版本

# kubectl version 
Client Version: version.Info{Major:"1", Minor:"11+", GitVersion:"v1.11.0-168+f47446a730ca03", GitCommit:"f47446a730ca037473fb3bf0c5abeea648c1ac12", GitTreeState:"clean", BuildDate:"2018-08-25T21:05:52Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"11+", GitVersion:"v1.11.0-168+f47446a730ca03", GitCommit:"f47446a730ca037473fb3bf0c5abeea648c1ac12", GitTreeState:"clean", BuildDate:"2018-08-25T21:05:52Z", GoVersion:"go1.10.3", Compiler:"gc", Platform:"linux/amd64"}

kube-proxy 是 kubernetes 重要的组建，它的作用就是虚拟出一个VIP(service),保证VIP无论后台服务(POD,Endpoint)如何变更都保持不变,起到一个LB的功能
kube-proxy提供了三种LB模式:
一种是基于用户态的模式userspace, 一种是iptables模式， 一种是ipvs模式

所以要分析细节，读者必须要了解iptables,ipvs的相关知识，这个读者自行百度了解，在这里我就不详细说了

---

不多说废话了，开始步入源代码
入口函数在$GOPATH/src/k8s.io/kubernetes/cmd/kube-proxy/proxy.go文件中,如下

接着分析函数command := app.NewProxyCommand()

// NewProxyCommand creates a *cobra.Command object with default parameters
func NewProxyCommand() *cobra.Command {
	opts := NewOptions()//这个函数主要是配置文件， KubeProxyConfiguration 这个结构体里接收启动时所传入的参数

	cmd := &cobra.Command{
		Use: "kube-proxy",
		Long: `The Kubernetes network proxy runs on each node. This
reflects services as defined in the Kubernetes API on each node and can do simple
TCP and UDP stream forwarding or round robin TCP and UDP forwarding across a set of backends.
Service cluster IPs and ports are currently found through Docker-links-compatible
environment variables specifying ports opened by the service proxy. There is an optional
addon that provides cluster DNS for these cluster IPs. The user must create a service
with the apiserver API to configure the proxy.`,
		Run: func(cmd *cobra.Command, args []string) {
			verflag.PrintAndExitIfRequested()
			utilflag.PrintFlags(cmd.Flags())
			//这个函数还没实现
			if err := initForOS(opts.WindowsService); err != nil {
				glog.Fatalf("failed OS init: %v", err)
			}

			//加载配置文件，获取传入的参数
			if err := opts.Complete(); err != nil {
				glog.Fatalf("failed complete: %v", err)
			}
			//校验kube-proxy的入参
			if err := opts.Validate(args); err != nil {
				glog.Fatalf("failed validate: %v", err)
			}
			glog.Fatal(opts.Run())
		},
	}

	var err error
	//设置kube-proxy的配置文件默认值
	opts.config, err = opts.ApplyDefaults(opts.config)
	if err != nil {
		glog.Fatalf("unable to create flag defaults: %v", err)
	}

	opts.AddFlags(cmd.Flags())
	//kube-proxy的配置文件类型
	cmd.MarkFlagFilename("config", "yaml", "yml", "json")

	return cmd
}

接下来查看opts.Run() 的具体实现
opts.Run()函数在$GOPATH/src/k8s.io/kubernetes/cmd/kube-proxy/app/server.go文件中，如下

writeConfigFile()函数的具体内容：该函数主要是加载配置文件

func (o *Options) writeConfigFile() error {
	var encoder runtime.Encoder
	mediaTypes := o.codecs.SupportedMediaTypes()
	for _, info := range mediaTypes {
		if info.MediaType == "application/yaml" {
			encoder = info.Serializer
			break
		}
	}
	if encoder == nil {
		return errors.New("unable to locate yaml encoder")
	}
	encoder = json.NewYAMLSerializer(json.DefaultMetaFactory, o.scheme, o.scheme)
	encoder = o.codecs.EncoderForVersion(encoder, v1alpha1.SchemeGroupVersion)

	configFile, err := os.Create(o.WriteConfigTo)
	if err != nil {
		return err
	}
	defer configFile.Close()

	if err := encoder.Encode(o.config, configFile); err != nil {
		return err
	}

	glog.Infof("Wrote configuration to: %s\n", o.WriteConfigTo)

	return nil
}

查看NewProxyServer函数,该函数是由函数newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master)具体实现
newProxyServer主要是根据proxyMode代理模式各自加载相关的各类属性以及管理所需要的工具等

$GOPATH/src/k8s.io/kubernetes/cmd/kube-proxy/app/server_others.go
具体细节如下

func newProxyServer(
	config *proxyconfigapi.KubeProxyConfiguration,
	cleanupAndExit bool,
	cleanupIPVS bool,
	scheme *runtime.Scheme,
	master string) (*ProxyServer, error) {

	if config == nil {
		return nil, errors.New("config is required")
	}

	if c, err := configz.New(proxyconfigapi.GroupName); err == nil {
		c.Set(config)
	} else {
		return nil, fmt.Errorf("unable to register configz: %s", err)
	}

	protocol := utiliptables.ProtocolIpv4
	if net.ParseIP(config.BindAddress).To4() == nil {
		glog.V(0).Infof("IPv6 bind address (%s), assume IPv6 operation", config.BindAddress)
		protocol = utiliptables.ProtocolIpv6
	}

	var iptInterface utiliptables.Interface
	var ipvsInterface utilipvs.Interface
	var kernelHandler ipvs.KernelHandler
	var ipsetInterface utilipset.Interface
	var dbus utildbus.Interface

	// Create a iptables utils.
	execer := exec.New()

	dbus = utildbus.New()
	iptInterface = utiliptables.New(execer, dbus, protocol)
	kernelHandler = ipvs.NewLinuxKernelHandler()
	ipsetInterface = utilipset.New(execer)
	if canUse, _ := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface); canUse {
		ipvsInterface = utilipvs.New(execer)
	}

	// We omit creation of pretty much everything if we run in cleanup mode
	if cleanupAndExit {
		return &ProxyServer{
			execer:         execer,
			IptInterface:   iptInterface,
			IpvsInterface:  ipvsInterface,
			IpsetInterface: ipsetInterface,
			CleanupAndExit: cleanupAndExit,
		}, nil
	}

	client, eventClient, err := createClients(config.ClientConnection, master)
	if err != nil {
		return nil, err
	}

	// Create event recorder
	hostname := utilnode.GetHostname(config.HostnameOverride)
	eventBroadcaster := record.NewBroadcaster()
	recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname})

	nodeRef := &v1.ObjectReference{
		Kind:      "Node",
		Name:      hostname,
		UID:       types.UID(hostname),
		Namespace: "",
	}

	var healthzServer *healthcheck.HealthzServer
	var healthzUpdater healthcheck.HealthzUpdater
	//启动健康检查端口
	if len(config.HealthzBindAddress) > 0 {
		healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef)
		healthzUpdater = healthzServer
	}

	var proxier proxy.ProxyProvider
	//Service Endpoint的CRDU
	var serviceEventHandler proxyconfig.ServiceHandler
	var endpointsEventHandler proxyconfig.EndpointsHandler

	proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
	//iptables模式
	if proxyMode == proxyModeIPTables {
		glog.V(0).Info("Using iptables Proxier.")
		nodeIP := net.ParseIP(config.BindAddress)
		if nodeIP.Equal(net.IPv4zero) || nodeIP.Equal(net.IPv6zero) {
			nodeIP = getNodeIP(client, hostname)
		}
		if config.IPTables.MasqueradeBit == nil {
			// MasqueradeBit must be specified or defaulted.
			return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config")
		}

		// TODO this has side effects that should only happen when Run() is invoked.
		proxierIPTables, err := iptables.NewProxier(
			iptInterface,
			utilsysctl.New(),
			execer,
			config.IPTables.SyncPeriod.Duration,
			config.IPTables.MinSyncPeriod.Duration,
			config.IPTables.MasqueradeAll,
			int(*config.IPTables.MasqueradeBit),
			config.ClusterCIDR,
			hostname,
			nodeIP,
			recorder,
			healthzUpdater,
			config.NodePortAddresses,
		)
		if err != nil {
			return nil, fmt.Errorf("unable to create proxier: %v", err)
		}
		metrics.RegisterMetrics()
		proxier = proxierIPTables
		serviceEventHandler = proxierIPTables
		endpointsEventHandler = proxierIPTables
		// No turning back. Remove artifacts that might still exist from the userspace Proxier.
		glog.V(0).Info("Tearing down inactive rules.")
		// TODO this has side effects that should only happen when Run() is invoked.
		userspace.CleanupLeftovers(iptInterface)
		// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
		// Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given
		// ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush
		// all ipvs rules when kube-proxy start up.  Users do this operation should be with caution.
		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
		//IPVS 模式
	} else if proxyMode == proxyModeIPVS {
		glog.V(0).Info("Using ipvs Proxier.")
		proxierIPVS, err := ipvs.NewProxier(
			iptInterface,
			ipvsInterface,
			ipsetInterface,
			utilsysctl.New(),
			execer,
			config.IPVS.SyncPeriod.Duration,
			config.IPVS.MinSyncPeriod.Duration,
			config.IPVS.ExcludeCIDRs,
			config.IPTables.MasqueradeAll,
			int(*config.IPTables.MasqueradeBit),
			config.ClusterCIDR,
			hostname,
			getNodeIP(client, hostname),
			recorder,
			healthzServer,
			config.IPVS.Scheduler,
			config.NodePortAddresses,
		)
		if err != nil {
			return nil, fmt.Errorf("unable to create proxier: %v", err)
		}
		metrics.RegisterMetrics()
		proxier = proxierIPVS
		serviceEventHandler = proxierIPVS
		endpointsEventHandler = proxierIPVS
		glog.V(0).Info("Tearing down inactive rules.")
		// TODO this has side effects that should only happen when Run() is invoked.
		userspace.CleanupLeftovers(iptInterface)
		iptables.CleanupLeftovers(iptInterface)
		//用户空间
	} else {
		glog.V(0).Info("Using userspace Proxier.")
		// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
		// our config.EndpointsConfigHandler.
		loadBalancer := userspace.NewLoadBalancerRR()
		// set EndpointsConfigHandler to our loadBalancer
		endpointsEventHandler = loadBalancer

		// TODO this has side effects that should only happen when Run() is invoked.
		proxierUserspace, err := userspace.NewProxier(
			loadBalancer,
			net.ParseIP(config.BindAddress),
			iptInterface,
			execer,
			*utilnet.ParsePortRangeOrDie(config.PortRange),
			config.IPTables.SyncPeriod.Duration,
			config.IPTables.MinSyncPeriod.Duration,
			config.UDPIdleTimeout.Duration,
			config.NodePortAddresses,
		)
		if err != nil {
			return nil, fmt.Errorf("unable to create proxier: %v", err)
		}
		serviceEventHandler = proxierUserspace
		proxier = proxierUserspace

		// Remove artifacts from the iptables and ipvs Proxier, if not on Windows.
		glog.V(0).Info("Tearing down inactive rules.")
		// TODO this has side effects that should only happen when Run() is invoked.
		iptables.CleanupLeftovers(iptInterface)
		// IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode.
		// Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given
		// ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush
		// all ipvs rules when kube-proxy start up.  Users do this operation should be with caution.
		ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS)
	}

	iptInterface.AddReloadFunc(proxier.Sync)

	return &ProxyServer{
		Client:                 client,
		EventClient:            eventClient,
		IptInterface:           iptInterface,
		IpvsInterface:          ipvsInterface,
		IpsetInterface:         ipsetInterface,
		execer:                 execer,
		Proxier:                proxier,
		Broadcaster:            eventBroadcaster,
		Recorder:               recorder,
		ConntrackConfiguration: config.Conntrack,
		Conntracker:            &realConntracker{},
		ProxyMode:              proxyMode,
		NodeRef:                nodeRef,
		MetricsBindAddress:     config.MetricsBindAddress,
		EnableProfiling:        config.EnableProfiling,
		OOMScoreAdj:            config.OOMScoreAdj,
		ResourceContainer:      config.ResourceContainer,
		ConfigSyncPeriod:       config.ConfigSyncPeriod.Duration,
		ServiceEventHandler:    serviceEventHandler,
		EndpointsEventHandler:  endpointsEventHandler,
		HealthzServer:          healthzServer,
	}, nil
}

由于代码过长不好分析，接下来我们分开分析，首先查看返回值*ProxyServer的属性如下

// ProxyServer represents all the parameters required to start the Kubernetes proxy server. All
// fields are required.
type ProxyServer struct {
	Client                 clientset.Interface//连接kube-apiserver的对象
	EventClient            v1core.EventsGetter //事件获取方法
	IptInterface           utiliptables.Interface//操作iptables的函数接口(CRUD)
	IpvsInterface          utilipvs.Interface//操作ipvs的函数接口(CRUD)
	IpsetInterface         utilipset.Interface//操作ipset的函数接口(CRUD)
	execer                 exec.Interface//命令执行接口
	Proxier                proxy.ProxyProvider//同步工具
	Broadcaster            record.EventBroadcaster//事件广播器
	Recorder               record.EventRecorder//
	ConntrackConfiguration kubeproxyconfig.KubeProxyConntrackConfiguration//链路工具的配置信息
	Conntracker            Conntracker // if nil, ignored
	ProxyMode              string//代理模式
	NodeRef                *v1.ObjectReference
	CleanupAndExit         bool
	CleanupIPVS            bool
	MetricsBindAddress     string
	EnableProfiling        bool
	OOMScoreAdj            *int32
	ResourceContainer      string
	ConfigSyncPeriod       time.Duration
	ServiceEventHandler    config.ServiceHandler
	EndpointsEventHandler  config.EndpointsHandler
	HealthzServer          *healthcheck.HealthzServer
}

	// Create a iptables utils.
	execer := exec.New()//创建执行器executor实现Interface,$GOPATH/src/k8s.io/kubernetes/vendor/k8s.io/utils/exec/exec.go

	dbus = utildbus.New()//dbusImpl{}实现Interface方法  $GOPATH/src/k8s.io/kubernetes/pkg/util/dbus/dbus.go
	iptInterface = utiliptables.New(execer, dbus, protocol)// $GOPATH /src/k8s.io/kubernetes/pkg/util/iptables/iptables.go runner结构体实现了Interface
	
	kernelHandler = ipvs.NewLinuxKernelHandler()//主要检查内核模块以及内核相关的参数，$GOPATH/src/k8s.io/kubernetes/pkg/proxy/ipvs/proxier.go
	ipsetInterface = utilipset.New(execer)// runner结构实现Interface $GOPATH/src/k8s.io/kubernetes/pkg/util/ipset/ipset.go
	//检查内核参数
	if canUse, _ := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface); canUse {
		ipvsInterface = utilipvs.New(execer)//ipvsadm的使用方法
	}

CanUseIPVSProxier函数的实现$GOPATH/src/k8s.io/kubernetes/pkg/proxy/ipvs/proxier.go

func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) {
	mods, err := handle.GetModules()//获取内核模块
	if err != nil {
		return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
	}
	//golang版本的set
	wantModules := sets.NewString()
	loadModules := sets.NewString()
	wantModules.Insert(ipvsModules...)
	loadModules.Insert(mods...)
	modules := wantModules.Difference(loadModules).UnsortedList()
	if len(modules) != 0 {
		return false, fmt.Errorf("IPVS proxier will not be used because the following required kernel modules are not loaded: %v", modules)
	}

	// Check ipset version
	versionString, err := ipsetver.GetVersion()
	if err != nil {
		return false, fmt.Errorf("error getting ipset version, error: %v", err)
	}
	if !checkMinVersion(versionString) {
		return false, fmt.Errorf("ipset version: %s is less than min required version: %s", versionString, MinIPSetCheckVersion)
	}
	return true, nil
}

获取连接kube-apiservet组件的对象

...
client, eventClient, err := createClients(config.ClientConnection, maste