k8s源码分析-----kubelet协程分析

(1) 垃圾筒收集协程

在KubeletConfig CreateAndInitKubelet函数中，调用k.StartGarbageCollection()启动

k8s.io\kubernetes\cmd\kubelet\app\server.go:982

// Starts garbage collection threads.
func (kl *Kubelet) StartGarbageCollection() {
       go wait.Until(func() {
              if err := kl.containerGC.GarbageCollect(kl.sourcesReady.AllReady()); err != nil {
                     glog.Errorf("Container garbage collection failed: %v", err)
              }
       }, ContainerGCPeriod, wait.NeverStop)

       go wait.Until(func() {
              if err := kl.imageManager.GarbageCollect(); err != nil {
                     glog.Errorf("Image garbage collection failed: %v", err)
              }
       }, ImageGCPeriod, wait.NeverStop)
}

(2) initializeModule函数中启动了四个协程

func (kl *Kubelet) Run 调用initializeModules -》 imageManager.Start

k8s.io/kubernetes/pkg/kubelet.(*realImageManager).Start
k8s.io/kubernetes/pkg/kubelet/cm.(*containerManagerImpl).Start
k8s.io/kubernetes/pkg/kubelet.(*realOOMWatcher).Start
k8s.io/kubernetes/pkg/kubelet/server/stats.(*fsResourceAnalyzer).Start

// initializeModules will initialize internal modules that do not require the container runtime to be up.
// Note that the modules here must not depend on modules that are not initialized here.
func (kl *Kubelet) initializeModules() error {
       // Step 1: Promethues metrics.
       metrics.Register(kl.runtimeCache)

       // Step 2: Setup filesystem directories.
       if err := kl.setupDataDirs(); err != nil {
              return err
       }

       // Step 3: If the container logs directory does not exist, create it.
       if _, err := os.Stat(containerLogsDir); err != nil {
              if err := kl.os.MkdirAll(containerLogsDir, 0755); err != nil {
                     glog.Errorf("Failed to create directory %q: %v", containerLogsDir, err)
              }
       }

       // Step 4: Start the image manager.
       if err := kl.imageManager.Start(); err != nil {
              return fmt.Errorf("Failed to start ImageManager, images may not be garbage collected: %v", err)
       }

       // Step 5: Start container manager.
       if err := kl.containerManager.Start(); err != nil {
              return fmt.Errorf("Failed to start ContainerManager %v", err)
       }

       // Step 6: Start out of memory watcher.
       if err := kl.oomWatcher.Start(kl.nodeRef); err != nil {
              return fmt.Errorf("Failed to start OOM watcher %v", err)
       }

       // Step 7: Start resource analyzer
       kl.resourceAnalyzer.Start()

       return nil
}

（3）Kubelet Run函数中启动的协程

Kubelet Run函数中启动大多数的协程

// Run starts the kubelet reacting to config updates
func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
       if kl.logServer == nil {
              kl.logServer = http.StripPrefix("/logs/", http.FileServer(http.Dir("/var/log/")))
       }
       if kl.kubeClient == nil {
              glog.Warning("No api server defined - no node status update will be sent.")
       }
       if err := kl.initializeModules(); err != nil {
              kl.recorder.Eventf(kl.nodeRef, api.EventTypeWarning, kubecontainer.KubeletSetupFailed, err.Error())
              glog.Error(err)
              kl.runtimeState.setInitError(err)
       }

       // Start volume manager
       go kl.volumeManager.Run(wait.NeverStop)

       if kl.kubeClient != nil {
              // Start syncing node status immediately, this may set up things the runtime needs to run.
              go wait.Until(kl.syncNodeStatus, kl.nodeStatusUpdateFrequency, wait.NeverStop)
       }
       go wait.Until(kl.syncNetworkStatus, 30*time.Second, wait.NeverStop)
       go wait.Until(kl.updateRuntimeUp, 5*time.Second, wait.NeverStop)

       // Start a goroutine responsible for killing pods (that are not properly
       // handled by pod workers).
       go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop)

       // Start component sync loops.
       kl.statusManager.Start()
       kl.probeManager.Start()
       kl.evictionManager.Start(kl.getActivePods, evictionMonitoringPeriod)

       // Start the pod lifecycle event generator.
       kl.pleg.Start()
       kl.syncLoop(updates, kl)
}

(4)server.go的run函数中启动Healthz的协程

代码位置为：k8s.io\kubernetes\cmd\kubelet\app\server.go:396

在server.go的run函数中

if s.HealthzPort > 0 {
       healthz.DefaultHealthz()
       go wait.Until(func() {
              err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), nil)
              if err != nil {
                     glog.Errorf("Starting health server failed: %v", err)
              }
       }, 5*time.Second, wait.NeverStop)
}

（5） 日志flush的协程

main函数中： util.InitLogs()

// InitLogs initializes logs the way we want for kubernetes.
func InitLogs() {
       log.SetOutput(GlogWriter{})
       log.SetFlags(0)
       // The default glog flush interval is 30 seconds, which is frighteningly long.
       go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop)
}

(6) SourceFile携程

func NewSourceFile(path string, nodeName string, period time.Duration, updates chan<- interface{}) {
       config := &sourceFile{
              path:     path,
              nodeName: nodeName,
              updates:  updates,
       }
       glog.V(1).Infof("Watching path %q", path)
       go wait.Until(config.run, period, wait.NeverStop)
}

调用链为run-> RunKubelet -> CreateAndInitKubelet ->makePodSourceConfig->NewSourceFile

(7)  SourceURL 协程

func NewSourceURL(url string, header http.Header, nodeName string, period time.Duration, updates chan<- interface{}) {
       config := &sourceURL{
              url:      url,
              header:   header,
              nodeName: nodeName,
              updates:  updates,
              data:     nil,
              // Timing out requests leads to retries. This client is only used to
              // read the the manifest URL passed to kubelet.
              client: &http.Client{Timeout: 10 * time.Second},
       }
       glog.V(1).Infof("Watching URL %s", url)
       go wait.Until(config.run, period, wait.NeverStop)
}

调用链为run-> RunKubelet -> CreateAndInitKubelet ->makePodSourceConfig->NewSourceURL

(8)  APIService Watch协程

// NewSourceApiserver creates a config source that watches and pulls from the apiserver.
func NewSourceApiserver(c *clientset.Clientset, nodeName string, updates chan<- interface{}) {
       lw := cache.NewListWatchFromClient(c.CoreClient, "pods", api.NamespaceAll, fields.OneTermEqualSelector(api.PodHostField, nodeName))
       newSourceApiserverFromLW(lw, updates)
}

// newSourceApiserverFromLW holds creates a config source that watches and pulls from the apiserver.
func newSourceApiserverFromLW(lw cache.ListerWatcher, updates chan<- interface{}) {
       send := func(objs []interface{}) {
              var pods []*api.Pod
              for _, o := range objs {
                     pods = append(pods, o.(*api.Pod))
              }
              updates <- kubetypes.PodUpdate{Pods: pods, Op: kubetypes.SET, Source: kubetypes.ApiserverSource}
       }
       cache.NewReflector(lw, &api.Pod{}, cache.NewUndeltaStore(send, cache.MetaNamespaceKeyFunc), 0).Run()
}

// Run starts a watch and handles watch events. Will restart the watch if it is closed.
// Run starts a goroutine and returns immediately.
func (r *Reflector) Run() {
       glog.V(3).Infof("Starting reflector %v (%s) from %s", r.expectedType, r.resyncPeriod, r.name)
       go wait.Until(func() {
              if err := r.ListAndWatch(wait.NeverStop); err != nil {
                     utilruntime.HandleError(err)
              }
       }, r.period, wait.NeverStop)
}

调用链为run-> RunKubelet -> CreateAndInitKubelet ->makePodSourceConfig->NewSourceURL

(9)  start the kubelet server协程

该协程在Run函数的协程都启动完成之后启动

func startKubelet(k KubeletBootstrap, podCfg *config.PodConfig, kc *KubeletConfig) {
       // start the kubelet
       go wait.Until(func() { k.Run(podCfg.Updates()) }, 0, wait.NeverStop)

       // start the kubelet server
       if kc.EnableServer {
              go wait.Until(func() {
                     k.ListenAndServe(kc.Address, kc.Port, kc.TLSOptions, kc.Auth, kc.EnableDebuggingHandlers)
              }, 0, wait.NeverStop)
       }
       if kc.ReadOnlyPort > 0 {
              go wait.Until(func() {
                     k.ListenAndServeReadOnly(kc.Address, kc.ReadOnlyPort)
              }, 0, wait.NeverStop)
       }
}

// ListenAndServe runs the kubelet HTTP server.
func (kl *Kubelet) ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableDebuggingHandlers bool) {
       server.ListenAndServeKubeletServer(kl, kl.resourceAnalyzer, address, port, tlsOptions, auth, enableDebuggingHandlers, kl.containerRuntime)
}

// ListenAndServeKubeletServer initializes a server to respond to HTTP network requests on the Kubelet.
func ListenAndServeKubeletServer(
       host HostInterface,
       resourceAnalyzer stats.ResourceAnalyzer,
       address net.IP,
       port uint,
       tlsOptions *TLSOptions,
       auth AuthInterface,
       enableDebuggingHandlers bool,
       runtime kubecontainer.Runtime) {
       glog.Infof("Starting to listen on %s:%d", address, port)
       handler := NewServer(host, resourceAnalyzer, auth, enableDebuggingHandlers, runtime)
       s := &http.Server{
              Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
              Handler:        &handler,
              MaxHeaderBytes: 1 << 20,
       }
       if tlsOptions != nil {
              s.TLSConfig = tlsOptions.Config
              glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile))
       } else {
              glog.Fatal(s.ListenAndServe())
       }
}