kubernetes 1.24.2实战与源码
第7章 kube-controller-manager控制器管理中心的作用
7.1 controller-manager启动主流程
- controller-manager启动主流程
本节重点总结
controller-manager启动主流程
controller-manager代码入口
cobra.Command中的Run方法
D:\Workspace\Go\src\k8s.io\kubernetes\cmd\kube-controller-manager\app\controllermanager.go
Run: func(cmd *cobra.Command, args []string) {
verflag.PrintAndExitIfRequested()
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := s.Logs.ValidateAndApply(utilfeature.DefaultFeatureGate); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
cliflag.PrintFlags(cmd.Flags())
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
if err := Run(c.Complete(), wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
01 config创建cm的配置
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
Config传参解析 KnownControllers返回常用的控制器的名字
// KnownControllers returns all known controllers's name
func KnownControllers() []string {
ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops))
// add "special" controllers that aren't initialized normally. These controllers cannot be initialized
// using a normal function. The only known special case is the SA token controller which *must* be started
// first to ensure that the SA tokens for future controllers will exist. Think very carefully before adding
// to this list.
ret.Insert(
saTokenControllerName,
)
return ret.List()
}
根据NewControllerInitializers返回map的keys构建 string set
NewControllerInitializers返回常用控制器的名称和他们的初始化方法
// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["endpoint"] = startEndpointController
controllers["endpointslice"] = startEndpointSliceController
controllers["endpointslicemirroring"] = startEndpointSliceMirroringController
controllers["replicationcontroller"] = startReplicationController
controllers["podgc"] = startPodGCController
controllers["resourcequota"] = startResourceQuotaController
controllers["namespace"] = startNamespaceController
controllers["serviceaccount"] = startServiceAccountController
controllers["garbagecollector"] = startGarbageCollectorController
controllers["daemonset"] = startDaemonSetController
controllers["job"] = startJobController
controllers["deployment"] = startDeploymentController
controllers["replicaset"] = startReplicaSetController
controllers["horizontalpodautoscaling"] = startHPAController
controllers["disruption"] = startDisruptionController
controllers["statefulset"] = startStatefulSetController
controllers["cronjob"] = startCronJobController
controllers["csrsigning"] = startCSRSigningController
controllers["csrapproving"] = startCSRApprovingController
controllers["csrcleaner"] = startCSRCleanerController
controllers["ttl"] = startTTLController
controllers["bootstrapsigner"] = startBootstrapSignerController
controllers["tokencleaner"] = startTokenCleanerController
controllers["nodeipam"] = startNodeIpamController
controllers["nodelifecycle"] = startNodeLifecycleController
if loopMode == IncludeCloudLoops {
controllers["service"] = startServiceController
controllers["route"] = startRouteController
controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
// TODO: volume controller into the IncludeCloudLoops only set.
}
controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
controllers["attachdetach"] = startAttachDetachController
controllers["persistentvolume-expander"] = startVolumeExpandController
controllers["clusterrole-aggregation"] = startClusterRoleAggregrationController
controllers["pvc-protection"] = startPVCProtectionController
controllers["pv-protection"] = startPVProtectionController
controllers["ttl-after-finished"] = startTTLAfterFinishedController
controllers["root-ca-cert-publisher"] = startRootCACertPublisher
controllers["ephemeral-volume"] = startEphemeralVolumeController
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) &&
utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) {
controllers["storage-version-gc"] = startStorageVersionGCController
}
return controllers
}
saTokenControllerName 属于非常规初始化的控制器,单独Insert到set中
ret.Insert(
saTokenControllerName,
)
Config传参解析 ControllersDisabledByDefault代表默认禁止的控制器
// ControllersDisabledByDefault is the set of controllers which is disabled by default
var ControllersDisabledByDefault = sets.NewString(
"bootstrapsigner",
"tokencleaner",
)
Config内部解析
校验操作 D:\Workspace\Go\src\k8s.io\kubernetes\cmd\kube-controller-manager\app\options\options.go
if err := s.Validate(allControllers, disabledByDefaultControllers); err != nil {
return nil, err
}
创建kubeconfig和client
kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
if err != nil {
return nil, err
}
kubeconfig.DisableCompression = true
kubeconfig.ContentConfig.AcceptContentTypes = s.Generic.ClientConnection.AcceptContentTypes
kubeconfig.ContentConfig.ContentType = s.Generic.ClientConnection.ContentType
kubeconfig.QPS = s.Generic.ClientConnection.QPS
kubeconfig.Burst = int(s.Generic.ClientConnection.Burst)
client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, KubeControllerManagerUserAgent))
if err != nil {
return nil, err
}
创建事件记录器
eventRecorder := createRecorder(client, KubeControllerManagerUserAgent)
构造kubecontrollerconfig,然后ApplyTo应用
c := &kubecontrollerconfig.Config{
Client: client,
Kubeconfig: kubeconfig,
EventRecorder: eventRecorder,
}
if err := s.ApplyTo(c); err != nil {
return nil, err
}
使用kubecontrollerconfig初始化KubeControllerManagerOptions
·ApplyTo内部调用各个配置项的ApplyTo方法赋值,如StatefulSetController
if err := s.StatefulSetController.ApplyTo(&c.ComponentConfig.StatefulSetController); err != nil {
return err
}
原始的s来自于NewControllerManagerCommand中
使用默认的配置构造KubeControllerManagerOptions
s, err := options.NewKubeControllerManagerOptions()
NewDefaultComponentConfig函数返回默认的配置,位置D:\Workspace\Go\src\k8s.io\kubernetes\cmd\kube-controller-manager\app\options\options.go
// NewDefaultComponentConfig returns kube-controller manager configuration object.
func NewDefaultComponentConfig() (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) {
versioned := kubectrlmgrconfigv1alpha1.KubeControllerManagerConfiguration{}
kubectrlmgrconfigscheme.Scheme.Default(&versioned)
internal := kubectrlmgrconfig.KubeControllerManagerConfiguration{}
if err := kubectrlmgrconfigscheme.Scheme.Convert(&versioned, &internal, nil); err != nil {
return internal, err
}
return internal, nil
}
在NewKubeControllerManagerOptions中可以看到,s的各个controller字段使用componentConfig赋值
s := KubeControllerManagerOptions{
Generic: cmoptions.NewGenericControllerManagerConfigurationOptions(&componentConfig.Generic),
KubeCloudShared: cpoptions.NewKubeCloudSharedOptions(&componentConfig.KubeCloudShared),
ServiceController: &cpoptions.ServiceControllerOptions{
ServiceControllerConfiguration: &componentConfig.ServiceController,
},
AttachDetachController: &AttachDetachControllerOptions{
&componentConfig.AttachDetachController,
},
CSRSigningController: &CSRSigningControllerOptions{
&componentConfig.CSRSigningController,
},
DaemonSetController: &DaemonSetControllerOptions{
&componentConfig.DaemonSetController,
},
DeploymentController: &DeploymentControllerOptions{
&componentConfig.DeploymentController,
},
StatefulSetController: &StatefulSetControllerOptions{
&componentConfig.StatefulSetController,
},
DeprecatedFlags: &DeprecatedControllerOptions{
&componentConfig.DeprecatedController,
},
EndpointController: &EndpointControllerOptions{
&componentConfig.EndpointController,
},
EndpointSliceController: &EndpointSliceControllerOptions{
&componentConfig.EndpointSliceController,
},
EndpointSliceMirroringController: &EndpointSliceMirroringControllerOptions{
&componentConfig.EndpointSliceMirroringController,
},
EphemeralVolumeController: &EphemeralVolumeControllerOptions{
&componentConfig.EphemeralVolumeController,
},
GarbageCollectorController: &GarbageCollectorControllerOptions{
&componentConfig.GarbageCollectorController,
},
HPAController: &HPAControllerOptions{
&componentConfig.HPAController,
},
JobController: &JobControllerOptions{
&componentConfig.JobController,
},
CronJobController: &CronJobControllerOptions{
&componentConfig.CronJobController,
},
NamespaceController: &NamespaceControllerOptions{
&componentConfig.NamespaceController,
},
NodeIPAMController: &NodeIPAMControllerOptions{
&componentConfig.NodeIPAMController,
},
NodeLifecycleController: &NodeLifecycleControllerOptions{
&componentConfig.NodeLifecycleController,
},
PersistentVolumeBinderController: &PersistentVolumeBinderControllerOptions{
&componentConfig.PersistentVolumeBinderController,
},
PodGCController: &PodGCControllerOptions{
&componentConfig.PodGCController,
},
ReplicaSetController: &ReplicaSetControllerOptions{
&componentConfig.ReplicaSetController,
},
ReplicationController: &ReplicationControllerOptions{
&componentConfig.ReplicationController,
},
ResourceQuotaController: &ResourceQuotaControllerOptions{
&componentConfig.ResourceQuotaController,
},
SAController: &SAControllerOptions{
&componentConfig.SAController,
},
TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{
&componentConfig.TTLAfterFinishedController,
},
SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(),
Metrics: metrics.NewOptions(),
Logs: logs.NewOptions(),
}
02 run执行KubeControllerManagerOptions
if err := Run(c.Complete(), wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
初始化configz
ConfigzName kubecontrollermanager.config.k8s.io代表配置的map 外层key的名字
if cfgz, err := configz.New(ConfigzName); err == nil {
cfgz.Set(c.ComponentConfig)
} else {
klog.Errorf("unable to register configz: %v", err)
}
可以使用curl获取配置,其中可以看到具体的控制器的配置段,token是kube-system下的,来自于之前的prometheus token
![](https://i-blog.csdnimg.cn/blog_migrate/3fd0229abaa702e57aae37dd060cfec0.png)
健康监测与http服务
// Setup any healthz checks we will want to use.
var checks []healthz.HealthChecker
var electionChecker *leaderelection.HealthzAdaptor
if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
checks = append(checks, electionChecker)
}
healthzHandler := controllerhealthz.NewMutableHealthzHandler(checks...)
// Start the controller manager HTTP server
// unsecuredMux is the handler for these controller *after* authn/authz filters have been applied
var unsecuredMux *mux.PathRecorderMux
if c.SecureServing != nil {
unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, healthzHandler)
handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
// TODO: handle stoppedCh and listenerStoppedCh returned by c.SecureServing.Serve
if _, _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
return err
}
}
核心的启动方法
run := func(ctx context.Context, startSATokenController InitFunc, initializersFunc ControllerInitializersFunc) {
controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done())
if err != nil {
klog.Fatalf("error building controller context: %v", err)
}
controllerInitializers := initializersFunc(controllerContext.LoopMode)
if err := StartControllers(ctx, controllerContext, startSATokenController, controllerInitializers, unsecuredMux, healthzHandler); err != nil {
klog.Fatalf("error starting controllers: %v", err)
}
controllerContext.InformerFactory.Start(stopCh)
controllerContext.ObjectOrMetadataInformerFactory.Start(stopCh)
close(controllerContext.InformersStarted)
select {}
}
正常的话开启leader选举
通过抢锁成功后,触发OnStartedLeading回调,内部调用run方法
// Start the main lock
go leaderElectAndRun(c, id, electionChecker,
c.ComponentConfig.Generic.LeaderElection.ResourceLock,
c.ComponentConfig.Generic.LeaderElection.ResourceName,
leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
initializersFunc := NewControllerInitializers
if leaderMigrator != nil {
// If leader migration is enabled, we should start only non-migrated controllers
// for the main lock.
initializersFunc = createInitializersFunc(leaderMigrator.FilterFunc, leadermigration.ControllerNonMigrated)
klog.Info("leader migration: starting main controllers.")
}
run(ctx, startSATokenController, initializersFunc)
},
OnStoppedLeading: func() {
klog.ErrorS(nil, "leaderelection lost")
klog.FlushAndExit(klog.ExitFlushTimeout, 1)
},
})
StartControllers启动所有注册的控制器
首先启动特殊的sa控制器因为其他控制器需要sa控制器构造token
// Always start the SA token controller first using a full-power client, since it needs to mint tokens for the rest
// If this fails, just return here and fail since other controllers won't be able to get credentials.
if startSATokenController != nil {
if _, _, err := startSATokenController(ctx, controllerCtx); err != nil {
return err
}
}
其次遍历注册控制器,执行他们的initFn即可
for controllerName, initFn := range controllers {
if !controllerCtx.IsControllerEnabled(controllerName) {
klog.Warningf("%q is disabled", controllerName)
continue
}
time.Sleep(wait.Jitter(controllerCtx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
klog.V(1).Infof("Starting %q", controllerName)
ctrl, started, err := initFn(ctx, controllerCtx)
if err != nil {
klog.Errorf("Error starting %q", controllerName)
return err
}
if !started {
klog.Warningf("Skipping %q", controllerName)
continue
}
check := controllerhealthz.NamedPingChecker(controllerName)
if ctrl != nil {
// check if the controller supports and requests a debugHandler
// and it needs the unsecuredMux to mount the handler onto.
if debuggable, ok := ctrl.(controller.Debuggable); ok && unsecuredMux != nil {
if debugHandler := debuggable.DebuggingHandler(); debugHandler != nil {
basePath := "/debug/controllers/" + controllerName
unsecuredMux.UnlistedHandle(basePath, http.StripPrefix(basePath, debugHandler))
unsecuredMux.UnlistedHandlePrefix(basePath+"/", http.StripPrefix(basePath, debugHandler))
}
}
if healthCheckable, ok := ctrl.(controller.HealthCheckable); ok {
if realCheck := healthCheckable.HealthChecker(); realCheck != nil {
check = controllerhealthz.NamedHealthChecker(controllerName, realCheck)
}
}
}
controllerChecks = append(controllerChecks, check)
klog.Infof("Started %q", controllerName)
}
7.2 ReplicaSet和对应的ReplicaSetController控制器
控制器模式
各种资源的控制器对比期望数量和当前运行数量
如果运行的不足就扩容
如果运行的多了就缩容
- 重启次数多的
渐进式扩容
-扩容采用渐进式扩容机制,从1开始2倍增加数量,直到扩容完毕
-目的是为了防止全量扩容时,同一个pod出现大量相同的错误,并且对服务组件压力较大
缩容有对应的筛选机制
缩容有对应的筛选机制,对运行的pod进行排序:目的是尽量删除哪些最近创建的,状态不稳定的pod
- Pending的
- 同一个节点上多开的
创建时间早的,ready时间短的
重启次数多的
本节重点总结:
各种资源的控制器对比期望数量和当前运行数量
。如果运行的不足就扩容
。如果运行的多了就缩容
获取当前数量通过syncHandler调用informer获取指定ns中的对象,再根据标签过滤
扩容采用渐进式扩容机制,从1开始2倍增加数量,指导扩容完毕
。目的是为了防止全量扩容时,同一个pod出现大量相同的错误,并且对服务组件压力较大
缩容有对应的筛选机制,对运行的pod进行排序: 目的是尽量删除哪些最近创建的,状态不定的pod
。Pending的
。同一个节点上多开的
。创建时间早的,ready时间短的
。重启次数多的
什么是replicaset
文档地址 https://kubernetes.io/zh/docs/concepts/workloads/controllers/replicaset/
ReplicaSet的目的是维护一组在任何时候都处于运行状态的 Pod副本的稳定集合。
因此,它通常用来保证给定数量的、完全相同的 Pod 的可用性
ReplicaSet 的工作原理
它对比了定义声明的Pod数和当前集群中满足条件的Pod数,进行相对应的扩缩容
replicaset控制器源码解读
控制器map入口,位置D:\Workspace\Go\src\k8s.io\kubernetes\cmd\kube-controller-manager\app\controllermanager.go
// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
controllers := map[string]InitFunc{}
...
controllers["replicaset"] = startReplicaSetController
...
}
startReplicaSetController的准备工作
D:\Workspace\Go\src\k8s.io\kubernetes\cmd\kube-controller-manager\app\apps.go
使用Pods和ReplicaSets两个informer初始化rs控制器,因为需要同时观察这两种资源的数量
func startReplicaSetController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
go replicaset.NewReplicaSetController(
controllerContext.InformerFactory.Apps().V1().ReplicaSets(),
controllerContext.InformerFactory.Core().V1().Pods(),
controllerContext.ClientBuilder.ClientOrDie("replicaset-controller"),
replicaset.BurstReplicas,
).Run(ctx, int(controllerContext.ComponentConfig.ReplicaSetController.ConcurrentRSSyncs))
return nil, true, nil
}
NewReplicaSetController 使用event recorder初始化控制器
D:\Workspace\Go\src\k8s.io\kubernetes\pkg\controller\replicaset\replica_set.go
// NewReplicaSetController configures a replica set controller with the specified event recorder
func NewReplicaSetController(rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, kubeClient clientset.Interface, burstReplicas int) *ReplicaSetController {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartStructuredLogging(0)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
if err := metrics.Register(legacyregistry.Register); err != nil {
klog.ErrorS(err, "unable to register metrics")
}
return NewBaseController(rsInformer, podInformer, kubeClient, burstReplicas,
apps.SchemeGroupVersion.WithKind("ReplicaSet"),
"replicaset_controller",
"replicaset",
controller.RealPodControl{
KubeClient: kubeClient,
Recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "replicaset-controller"}),
},
)
}
底层调用NewBaseController
添加rsInformer的回调,对应rs对象新增、更新、删除
rsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: rsc.addRS,
UpdateFunc: rsc.updateRS,
DeleteFunc: rsc.deleteRS,
})
添加podinformer的回调,对应pod的新增、更新、删除
podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: rsc.addPod,
// This invokes the ReplicaSet for every pod change, eg: host assignment. Though this might seem like
// overkill the most frequent pod update is status, and the associated ReplicaSet will only list from
// local storage, so it should be ok.
UpdateFunc: rsc.updatePod,
DeleteFunc: rsc.deletePod,
})
举个rscaddRS的例子,底层就是往队列中添加一条
func (rsc *ReplicaSetController) enqueueRS(rs *apps.ReplicaSet) {
key, err := controller.KeyFunc(rs)
if err != nil {
utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", rs, err))
return
}
rsc.queue.Add(key)
}
设置syncHandler为syncReplicaSet
rsc.syncHandler = rsc.syncReplicaSet
startReplicaSetController的Run方法
并发同步的限制个数
位置D:\Workspace\Go\src\k8s.io\kubernetes\pkg\controller\replicaset\config\types.go
传入的ctx.ComponentConfig.ReplicaSetController.ConcurrentRSSync代表并发同步的限制个数
数量越高说明需要的cpu和网络等资源越高
// ReplicaSetControllerConfiguration contains elements describing ReplicaSetController.
type ReplicaSetControllerConfiguration struct {
// concurrentRSSyncs is the number of replica sets that are allowed to sync
// concurrently. Larger number = more responsive replica management, but more
// CPU (and network) load.
ConcurrentRSSyncs int32
}
对应的参数为--concurrent-replicaset-syncs
从configz可以看出默认值是5
通过rsc.podListerSynced判断informer数据已经同步过了
目的是在进行主流程钱,本地informer缓存中已经有数据了
if !cache.WaitForNamedCacheSync(rsc.Kind, ctx.Done(), rsc.podListerSynced, rsc.rsListerSynced) {
return
}
对应的判断方法在NewBaseController中赋值
rsc.podListerSynced podInformer.Informer().HasSynced
worker函数
。启动等于并发限制个数的worker
for i := 0; i < workers; i++ {
go wait.UntilWithContext(ctx, rsc.worker, time.Second)
}
死循环处理
// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (rsc *ReplicaSetController) worker(ctx context.Context) {
for rsc.processNextWorkItem(ctx) {
}
}
处理函数processNextWorkltem
代码如下
func (rsc *ReplicaSetController) processNextWorkItem(ctx context.Context) bool {
key, quit := rsc.queue.Get()
if quit {
return false
}
defer rsc.queue.Done(key)
err := rsc.syncHandler(ctx, key.(string))
if err == nil {
rsc.queue.Forget(key)
return true
}
utilruntime.HandleError(fmt.Errorf("sync %q failed with %v", key, err))
rsc.queue.AddRateLimited(key)
return true
}
解读一下
。先从队列中获取一个元素 rsc.queue.Get
。然后调用syncHandler处理
。如果没出错,调用rsc.queue.Forget删除,意思是不再retry了
。否则,其他的worker还要拿到这个key进行重试
syncHandler就是syncReplicaSet
D:\Workspace\Go\src\k8s.io\kubernetes\pkg\controller\replicaset\replica_set.go
。首先从key中分隔出rs的namespace和name
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
return err
}
通过rs informer获取rs对象
rs, err := rsc.rsLister.ReplicaSets(namespace).Get(name)
if apierrors.IsNotFound(err) {
klog.V(4).Infof("%v %v has been deleted", rsc.Kind, key)
rsc.expectations.DeleteExpectations(key)
return nil
}
获取rs的selector,就是标签选择器
selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
utilruntime.HandleError(fmt.Errorf("error converting pod selector to selector for rs %v/%v: %v", namespace, name, err))
return nil
}
筛选出rs所在ns的所有pod
// list all pods to include the pods that don't match the rs`s selector
// anymore but has the stale controller ref.
// TODO: Do the List and Filter in a single pass, or use an index.
allPods, err := rsc.podLister.Pods(rs.Namespace).List(labels.Everything())
if err != nil {
return err
}
去掉inactive的pod
// Ignore inactive pods.
filteredPods := controller.FilterActivePods(allPods)
// NOTE: filteredPods are pointing to objects from cache - if you need to
// modify them, you need to copy it first.
filteredPods, err = rsc.claimPods(ctx, rs, selector, filteredPods)
if err != nil {
return err
}
管理的方法,下面详细讲
var manageReplicasErr error
if rsNeedsSync && rs.DeletionTimestamp == nil {
manageReplicasErr = rsc.manageReplicas(ctx, filteredPods, rs)
}
计算更新后的状态
newStatus := calculateStatus(rs, filteredPods, manageReplicasErr)
核心函数manageReplicas解析
根据函数注释可以知道,主要用来检查和更新rs,期间并不会修改flteredPods,如果出错就将rs重新入队做retry
// manageReplicas checks and updates replicas for the given ReplicaSet.
// Does NOT modify <filteredPods>.
// It will requeue the replica set in case of an error while creating/deleting pods.
用当前运行的pod个数减去rs中配置的副本数得到diff
diff := len(filteredPods) - int(*(rs.Spec.Replicas))
如果diff<0说明当前运行的小于期望的,需要扩容
如果diff>0说明当前运行的多于期望的,需要缩容
扩容过程
慢启动扩容函数 slowStartBatch
目的是为了防止大量创建pod出现相同的错误
func slowStartBatch(count int, initialBatchSize int, fn func() error) (int, error) {
remaining := count
successes := 0
for batchSize := integer.IntMin(remaining, initialBatchSize); batchSize > 0; batchSize = integer.IntMin(2*batchSize, remaining) {
errCh := make(chan error, batchSize)
var wg sync.WaitGroup
wg.Add(batchSize)
for i := 0; i < batchSize; i++ {
go func() {
defer wg.Done()
if err := fn(); err != nil {
errCh <- err
}
}()
}
wg.Wait()
curSuccesses := batchSize - len(errCh)
successes += curSuccesses
if len(errCh) > 0 {
return successes, <-errCh
}
remaining -= batchSize
}
return successes, nil
}
func getPodsToDelete(filteredPods, relatedPods []*v1.Pod, diff int) []*v1.Pod {
// No need to sort pods if we are about to delete all of them.
// diff will always be <= len(filteredPods), so not need to handle > case.
if diff < len(filteredPods) {
podsWithRanks := getPodsRankedByRelatedPodsOnSameNode(filteredPods, relatedPods)
sort.Sort(podsWithRanks)
reportSortingDeletionAgeRatioMetric(filteredPods, diff)
}
return filteredPods[:diff]
}
排序的依据在ActivePodsWithRanks的Less中,位置D:\Workspace\Go\src\k8s.io\kubernetes\pkg\controller\controller_utils.go
策略1 未分配到node的小于分配到node的
// 1. assigned < unassigned
if s[i].Spec.NodeName != s[j].Spec.NodeName && (len(s[i].Spec.NodeName) == 0 || len(s[j].Spec.NodeName) == 0) {
return len(s[i].Spec.NodeName) > 0
}
策略2 PodPending < PodUnknown < PodRunning
三种状态对应的数字为0,1,2
// 2. PodRunning < PodUnknown < PodPending
if s[i].Status.Phase != s[j].Status.Phase {
return podPhaseToOrdinal[s[i].Status.Phase] > podPhaseToOrdinal[s[j].Status.Phase]
}
策略3 Not ready <ready
if podutil.IsPodReady(s[i]) != podutil.IsPodReady(s[j]) {
return podutil.IsPodReady(s[i])
}
策略4 pod删除代价小的排在前面
删除代价 pod-deletion-cost 是1.21引入的功能.
代表一个int32数字,设置在Annotations中,数字越小删除的代价越小,删除代价小代表pod的利用率较低
同一应用的不同 Pods 可能其利用率是不同的。在对应用执行缩容操作时,可能 希望移除利用率较低的Pods。为了避免频繁更新 Pods
// 4. lower pod-deletion-cost < higher pod-deletion cost
if utilfeature.DefaultFeatureGate.Enabled(features.PodDeletionCost) {
pi, _ := helper.GetDeletionCostFromPodAnnotations(s.Pods[i].Annotations)
pj, _ := helper.GetDeletionCostFromPodAnnotations(s.Pods[j].Annotations)
if pi != pj {
return pi < pj
}
}
策略5节点上多开的排前面
// 5. Doubled up < not doubled up
// If one of the two pods is on the same node as one or more additional
// ready pods that belong to the same replicaset, whichever pod has more
// colocated ready pods is less
if s.Rank[i] != s.Rank[j] {
return s.Rank[i] > s.Rank[j]
}
策略6 ready时间短的排前面
// 6. Been ready for empty time < less time < more time
// If both pods are ready, the latest ready one is smaller
if podutil.IsPodReady(s.Pods[i]) && podutil.IsPodReady(s.Pods[j]) {
readyTime1 := podReadyTime(s.Pods[i])
readyTime2 := podReadyTime(s.Pods[j])
if !readyTime1.Equal(readyTime2) {
if !utilfeature.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
return afterOrZero(readyTime1, readyTime2)
} else {
if s.Now.IsZero() || readyTime1.IsZero() || readyTime2.IsZero() {
return afterOrZero(readyTime1, readyTime2)
}
rankDiff := logarithmicRankDiff(*readyTime1, *readyTime2, s.Now)
if rankDiff == 0 {
return s.Pods[i].UID < s.Pods[j].UID
}
return rankDiff < 0
}
}
}
策略7 重启次多的排前面
// 7. Pods with containers with higher restart counts < lower restart counts
if maxContainerRestarts(s.Pods[i]) != maxContainerRestarts(s.Pods[j]) {
return maxContainerRestarts(s.Pods[i]) > maxContainerRestarts(s.Pods[j])
}
策略8 创建时间短的排前面
// 8. Empty creation time pods < newer pods < older pods
if !s.Pods[i].CreationTimestamp.Equal(&s.Pods[j].CreationTimestamp) {
if !utilfeature.DefaultFeatureGate.Enabled(features.LogarithmicScaleDown) {
return afterOrZero(&s.Pods[i].CreationTimestamp, &s.Pods[j].CreationTimestamp)
} else {
if s.Now.IsZero() || s.Pods[i].CreationTimestamp.IsZero() || s.Pods[j].CreationTimestamp.IsZero() {
return afterOrZero(&s.Pods[i].CreationTimestamp, &s.Pods[j].CreationTimestamp)
}
rankDiff := logarithmicRankDiff(s.Pods[i].CreationTimestamp, s.Pods[j].CreationTimestamp, s.Now)
if rankDiff == 0 {
return s.Pods[i].UID < s.Pods[j].UID
}
return rankDiff < 0
}
}
缩容动作
调用 DeletePod执行
errCh := make(chan error, diff)
var wg sync.WaitGroup
wg.Add(diff)
for _, pod := range podsToDelete {
go func(targetPod *v1.Pod) {
defer wg.Done()
if err := rsc.podControl.DeletePod(ctx, rs.Namespace, targetPod.Name, rs); err != nil {
// Decrement the expected number of deletes because the informer won't observe this deletion
podKey := controller.PodKey(targetPod)
rsc.expectations.DeletionObserved(rsKey, podKey)
if !apierrors.IsNotFound(err) {
klog.V(2).Infof("Failed to delete %v, decremented expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name)
errCh <- err
}
}
}(pod)
}
wg.Wait()
第8章 kubelet节点上控制容器生命周期的管理者
8.1 kubelet启动主流程
8.2 kubelet节点自注册源码分析
8.3 基于NodeStatus和lease对象的心跳机制
8.4 syncLoop响应pod创建的过程
8.5 kubelet维护pod的内存管理器podManager源码解析
8.6 volumeManager中的desiredStateOfWorld理想状态解析
8.7volumeManager中的reconciler协调器解析
8.8 statusManager同步pod状态
8.9 probeManager监控pod中容器的监控状况
第9章 kubelet稳定性保证Eviction驱逐和oom
9.1 Kubelet Eviction驱逐解读
9.2 EvictionManager源码解读
9.3 容器qos和OOMScoreAdj的取值范围
9.4 oomWatcher管理器源码解析
第10章 kubelet中的cgroupManager解读
10.1 cgroup-v1原理介绍和golang代码体验cgroup的cpu和memory限制
10.2 cgourp-v2原理介绍
10.3 kubelet中的cgroupManager解析和节点qos顶级目录创建
10.4 containerManager应用之创建容器cgroup目录
第11章 kubelet中资源管理器cpuManager、memoryManager、deviceManager解读
11.1 TopologyManager分析
11.3写goland代码体会cpuset原理
11.4 kubelet中cpuManager解读
11.5 memoryManager原理简介
11.6 memoryManager源码阅读
11.7 device-plugins设备插件机制介绍
11.8 deviceManager源码解读
第12章 kubelet pleg对象和containerManager总结
12.1 kubelet pleg对象介绍和源码解读
12.2 kubelet containerManager源码解读
第13章 kubelet containerRuntime和sandbox容器
13.1 containerRuntime原理简介
13.2 kubelet containerRuntime接口定义和初始化
13.3 sandbox简介和podsandbox
13.4 containerRuntime创建sandbox源码阅读