kube Controller manager - resourcequota源码阅读 informer

背景

       KCM主要是围绕informer框架进行事件的异步处理，该框架在client-go包中，主要的入口文件shared_informer.go。网上资料较少可参考下:

      https://www.jianshu.com/p/d17f70369c35

       

这个是我自己整理的KCM 和 APi server 交互及处理流程

DeltaFIFO: 用来存放从apiserver watch到的资源对象。本身是一个队列，底层是一个字符串数组和map。

Relector: 用于和api server进行通信的客户端，提供listwatch功能机制。

HandleDelta 方法名：用来处理DeltaFIFO中的数据。

SharedProcessor：结构体名称，主要作用是用来管理多个processlistener。

processlistener： 结构体名称，每种资源都有自己的回调函数名，但是集成informer框架中会被进一步封装成listener。 

---

因为工作需要，需要了解K8s中resourcequota资源计算的方式。所以本文的目的主要就是：

     1. 了解Kube Controller Manager的代码框架

     2.  弄清楚resource quota的计算方式。

主要流程:

1. 启动KCM 连接api server
2. 连接成功后第一次全量同步目标资源(resource quota).
3. Watch Etcd资源变化。watch成功后将变化资源同步到队列中，同时开启5个worker 协程.
4. 将资源和cache中进行同步。
5. 将更新变化后的资源同步到api-server中。

---

controller-manager.go/main方法

package main   func main() {    rand.Seed(time.Now().UTC().UnixNano())        command := app.NewControllerManagerCommand()      // TODO: once we switch everything over to Cobra commands, we can go back to calling    // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the    // normalize func and add the go flag set by hand.    pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)    pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)    // utilflag.InitFlags()    logs.InitLogs()    defer logs.FlushLogs()      if err := command.Execute(); err != nil {       fmt.Fprintf(os.Stderr, "%v\n", err)       os.Exit(1)    } }

 

     

app..NewControllerManagerCommand()方法

 

func NewControllerManagerCommand() *cobra.Command {    s := options.NewKubeControllerManagerOptions()    cmd := &cobra.Command{       Use: "kube-controller-manager",       Long: `The Kubernetes controller manager is a daemon that embeds the core control loops shipped with Kubernetes. In applications of robotics and automation, a control loop is a non-terminating loop that regulates the state of the system. In Kubernetes, a controller is a control loop that watches the shared state of the cluster through the apiserver and makes changes attempting to move the current state towards the desired state. Examples of controllers that ship with Kubernetes today are the replication controller, endpoints controller, namespace controller, and serviceaccounts controller.`,       Run: func(cmd *cobra.Command, args []string) {          verflag.PrintAndExitIfRequested()          utilflag.PrintFlags(cmd.Flags())          // 1. 解析kcm已知的controller，将controoler名称作为参数传递给后面的函数          // 2. 新建一个 config结构体，并将option的默认配置赋值到config中。          // 3. 新建一个KubeConfig(restclient config)          c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())          if err != nil {             fmt.Fprintf(os.Stderr, "%v\n", err)             os.Exit(1)          }            if err := Run(c.Complete()); err != nil {             fmt.Fprintf(os.Stderr, "%v\n", err)             os.Exit(1)          }       },    }    s.AddFlags(cmd.Flags(), KnownControllers(), ControllersDisabledByDefault.List())      return cmd }

 

options.NewKubeControllerManagerOptions()

// k8s中命令行中的参数统一称之为options。主要还是初始化一些配置，连接端口等，这里主要secure端口443以及insecure端口10252 func NewKubeControllerManagerOptions() *KubeControllerManagerOptions {    componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureKubeControllerManagerPort)    s := KubeControllerManagerOptions{       // The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.       // Please make common changes there but put anything kube-controller specific here.       Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig),    }    s.Generic.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes"    s.Generic.SecureServing.ServerCert.PairName = "kube-controller-manager"    gcIgnoredResources := make([]componentconfig.GroupResource, 0, len(garbagecollector.DefaultIgnoredResources()))    for r := range garbagecollector.DefaultIgnoredResources() {       gcIgnoredResources = append(gcIgnoredResources, componentconfig.GroupResource{Group: r.Group, Resource: r.Resource})    }    s.Generic.ComponentConfig.GCIgnoredResources = gcIgnoredResources    // 直接将当前的kcm置成Leader。也可以看出一个集群内部只能存在一个KCM。    s.Generic.ComponentConfig.LeaderElection.LeaderElect = true    return &s }   controllerManager.go/Run()方法: func Run(c *config.CompletedConfig) error {    // To help debugging, immediately log version    glog.Infof("Version: %+v", version.Get())      // 新建的cfgz map将传进来的c保存起来。    if cfgz, err := configz.New("componentconfig"); err == nil {       cfgz.Set(c.Generic.ComponentConfig)    } else {       glog.Errorf("unable to register configz: %c", err)    }      // Start the controller manager HTTP Server using http2. tls1.2 协议    stopCh := make(chan struct{})    if c.Generic.SecureServing != nil {       if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh); err != nil {          return err       }    }    if c.Generic.InsecureServing != nil {       if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {          return err       }    }

   

   // 重点分析此run方法。

run := func(stop <-chan struct{}) {       rootClientBuilder := controller.SimpleControllerClientBuilder{          ClientConfig: c.Generic.Kubeconfig,       }       var clientBuilder controller.ControllerClientBuilder       if c.Generic.ComponentConfig.UseServiceAccountCredentials {          if len(c.Generic.ComponentConfig.ServiceAccountKeyFile) == 0 {             // It'c possible another controller process is creating the tokens for us.             // If one isn't, we'll timeout and exit when our client builder is unable to create the tokens.             glog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file")          }          clientBuilder = controller.SAControllerClientBuilder{             ClientConfig:         restclient.AnonymousClientConfig(c.Generic.Kubeconfig),             CoreClient:           c.Generic.Client.CoreV1(),             AuthenticationClient: c.Generic.Client.AuthenticationV1(),             Namespace:            "kube-system",          }       } else {          clientBuilder = rootClientBuilder       }       ctx, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, stop)       if err != nil {          glog.Fatalf("error building controller context: %v", err)       }       saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController         // 核心方法。运行各个controller       if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers(ctx.LoopMode)); err != nil {          glog.Fatalf("error starting controllers: %v", err)       }       ctx.InformerFactory.Start(ctx.Stop)       close(ctx.InformersStarted)       select {}    }      // 运行Run方法。    if !c.Generic.ComponentConfig.LeaderElection.LeaderElect {       run(wait.NeverStop)       panic("unreachable")    }    id, err := os.Hostname()    if err != nil {       return err    }      // add a uniquifier so that two processes on the same host don't accidentally both become active    id = id + "_" + string(uuid.NewUUID())    rl, err := resourcelock.New(c.Generic.ComponentConfig.LeaderElection.ResourceLock,       "kube-system",       "kube-controller-manager",       c.Generic.LeaderElectionClient.CoreV1(),       resourcelock.ResourceLockConfig{          Identity:      id,          EventRecorder: c.Generic.EventRecorder,       })    if err != nil {       glog.Fatalf("error creating lock: %v", err)    }      leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{       Lock:          rl,       LeaseDuration: c.Generic.ComponentConfig.LeaderElection.LeaseDuration.Duration,       RenewDeadline: c.Generic.ComponentConfig.LeaderElection.RenewDeadline.Duration,       RetryPeriod:   c.Generic.ComponentConfig.LeaderElection.RetryPeriod.Duration,</