kubernetes1.8 源码分析之资源调度

之前1.4的源码分析已经讲过调度的过程，现在载对一些细节补充一下。首先是k8s会加载哪些资源调度算法呢，其实它支持两种，一种是配置文件，一种是默认代码中指定的，下面看看plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go里面默认的调度算法，一个是筛选

func defaultPredicates() sets.String {
    return sets.NewString(
        // Fit is determined by volume zone requirements.
        factory.RegisterFitPredicateFactory(
            "NoVolumeZoneConflict",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                return predicates.NewVolumeZonePredicate(args.PVInfo, args.PVCInfo)
            },
        ),
        // Fit is determined by whether or not there would be too many AWS EBS volumes attached to the node
        factory.RegisterFitPredicateFactory(
            "MaxEBSVolumeCount",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
                maxVols := getMaxVols(aws.DefaultMaxEBSVolumes)
                return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
            },
        ),
        // Fit is determined by whether or not there would be too many GCE PD volumes attached to the node
        factory.RegisterFitPredicateFactory(
            "MaxGCEPDVolumeCount",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
                maxVols := getMaxVols(DefaultMaxGCEPDVolumes)
                return predicates.NewMaxPDVolumeCountPredicate(predicates.GCEPDVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
            },
        ),
        // Fit is determined by whether or not there would be too many Azure Disk volumes attached to the node
        factory.RegisterFitPredicateFactory(
            "MaxAzureDiskVolumeCount",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
                maxVols := getMaxVols(DefaultMaxAzureDiskVolumes)
                return predicates.NewMaxPDVolumeCountPredicate(predicates.AzureDiskVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
            },
        ),
        // Fit is determined by inter-pod affinity.
        factory.RegisterFitPredicateFactory(
            "MatchInterPodAffinity",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                return predicates.NewPodAffinityPredicate(args.NodeInfo, args.PodLister)
            },
        ),

        // Fit is determined by non-conflicting disk volumes.
        factory.RegisterFitPredicate("NoDiskConflict", predicates.NoDiskConflict),

        // GeneralPredicates are the predicates that are enforced by all Kubernetes components
        // (e.g. kubelet and all schedulers)
        factory.RegisterFitPredicate("GeneralPredicates", predicates.GeneralPredicates),

        // Fit is determined based on whether a pod can tolerate all of the node's taints
        factory.RegisterFitPredicate("PodToleratesNodeTaints", predicates.PodToleratesNodeTaints),

        // Fit is determined by node memory pressure condition.
        factory.RegisterFitPredicate("CheckNodeMemoryPressure", predicates.CheckNodeMemoryPressurePredicate),

        // Fit is determined by node disk pressure condition.
        factory.RegisterFitPredicate("CheckNodeDiskPressure", predicates.CheckNodeDiskPressurePredicate),

        // Fit is determined by node disk mount condition.
        factory.RegisterFitPredicate("CheckNodeDiskMountPressure", predicates.CheckNodeDiskMountPressurePredicate),

        // Fit is determined by volume zone requirements.
        factory.RegisterFitPredicateFactory(
            "NoVolumeNodeConflict",
            func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
                return predicates.NewVolumeNodePredicate(args.PVInfo, args.PVCInfo, nil)
            },
        ),
    )
}

筛选主要包括磁盘冲突、内存压力、节点状态，当然还包括节点端口标签选择等，
还有就是默认打分策略

func defaultPriorities() sets.String {
    return sets.NewString(
        // spreads pods by minimizing the number of pods (belonging to the same service or replication controller) on the same node.
        factory.RegisterPriorityConfigFactory(
            "SelectorSpreadPriority",
            factory.PriorityConfigFactory{
                Function: func(args factory.PluginFactoryArgs) algorithm.PriorityFunction {
                    return priorities.NewSelectorSpreadPriority(args.ServiceLister, args.ControllerLister, args.ReplicaSetLister, args.StatefulSetLister)
                },
                Weight: 1,
            },
        ),
        // pods should be placed in the same topological domain (e.g. same node, same rack, same zone, same power domain, etc.)
        // as some other pods, or, conversely, should not be placed in the same topological domain as some other pods.
        factory.RegisterPriorityConfigFactory(
            "InterPodAffinityPriority",
            factory.PriorityConfigFactory{
                Function: func(args factory.PluginFactoryArgs) algorithm.PriorityFunction {
                    return priorities.NewInterPodAffinityPriority(args.NodeInfo, args.NodeLister, args.PodLister, args.HardPodAffinitySymmetricWeight)
                },
                Weight: 1,
            },
        ),

        // Prioritize nodes by least requested utilization.
        factory.RegisterPriorityFunction2("LeastRequestedPriority", priorities.LeastRequestedPriorityMap, nil, 1),

        // Prioritizes nodes to help achieve balanced resource usage
        factory.RegisterPriorityFunction2("BalancedResourceAllocation", priorities.BalancedResourceAllocationMap, nil, 1),

        // Set this weight large enough to override all other priority functions.
        // TODO: Figure out a better way to do this, maybe at same time as fixing #24720.
        factory.RegisterPriorityFunction2("NodePreferAvoidPodsPriority", priorities.CalculateNodePreferAvoidPodsPriorityMap, nil, 10000),

        // Prioritizes nodes that have labels matching NodeAffinity
        factory.RegisterPriorityFunction2("NodeAffinityPriority", priorities.CalculateNodeAffinityPriorityMap, priorities.CalculateNodeAffinityPriorityReduce, 1),

        // TODO: explain what it does.
        factory.RegisterPriorityFunction2("TaintTolerationPriority", priorities.ComputeTaintTolerationPriorityMap, priorities.ComputeTaintTolerationPriorityReduce, 1),
    )
}

这个里面主要是一些资源优化调度保证资源平均使用以及标签和pod的亲和性关联等。当系统以及加载好这些过滤和打分的算法以后，那么当来了一个pod需要调度的时候，改怎么去调度呢？在说具体调度时候，先看看cache机制，这个和k8s其它组件里面的cache是一样的，避免频繁调用k8s 的apiserver。先看cache接口

type Cache interface {
    // AssumePod assumes a pod scheduled and aggregates the pod's information into its node.
    // The implementation also decides the policy to expire pod before being confirmed (receiving Add event).
    // After expiration, its information would be subtracted.
    AssumePod(pod *v1.Pod) error

    // FinishBinding signals that cache for assumed pod can be expired
    FinishBinding(pod *v1.Pod) error

    // ForgetPod removes an assumed pod from cache.
    ForgetPod(pod *v1.Pod) error

    // AddPod either confirms a pod if it's assumed, or adds it back if it's expired.
    // If added back, the pod's information would be added again.
    AddPod(pod *v1.Pod) error

    // UpdatePod removes oldPod's information and adds newPod's information.
    UpdatePod(oldPod, newPod *v1.Pod) error

    // RemovePod removes a pod. The pod's information would be subtracted from assigned node.
    RemovePod(pod *v1.Pod) error

    // AddNode adds overall information about node.
    AddNode(node *v1.Node) error

    // UpdateNode updates overall information about node.
    UpdateNode(oldNode, newNode *v1.Node) error

    // RemoveNode removes overall information about node.
    RemoveNode(node *v1.Node) error

    // UpdateNodeNameToInfoMap updates the passed infoMap to the current contents of Cache.
    // The node info contains aggregated information of pods scheduled (including assumed to be)
    // on this node.
    UpdateNodeNameToInfoMap(infoMap map[string]*NodeInfo) error

    // List lists all cached pods (including assumed ones).
    List(labels.Selector) ([]*v1.Pod, error)
}

包括了pod和node的一些方法，当watch这些资源变化时候就可以更新里面的数据，篇幅有限，那一个方法说一下，先看注册事件plugin/pkg/scheduler/factory/factory.go

    nodeInformer.Informer().AddEventHandlerWithResyncPeriod(
        cache.ResourceEventHandlerFuncs{
            AddFunc:    c.addNodeToCache,
            UpdateFunc: c.updateNodeInCache,
            DeleteFunc: c.deleteNodeFromCache,
        },
        0,
    )

上面的node add的事件将会出发cache的AddNode方法。

func (cache *schedulerCache) AddNode(node *v1.Node) error {
    cache.mu.Lock()
    defer cache.mu.Unlock()

    n, ok := cache.nodes[node.Name]
    if !ok {
        n = NewNodeInfo()
        cache.nodes[node.Name] = n
    }
    return n.SetNode(node)
}

那么cache里面就可以保存这个node信息了。通过处理保存到cache里面的数据

func (n *NodeInfo) SetNode(node *v1.Node) error {
    n.node = node
    for rName, rQuant := range node.Status.Allocatable {
        switch rName {
        case v1.ResourceCPU:
            n.allocatableResource.MilliCPU = rQuant.MilliValue()
        case v1.ResourceMemory:
            n.allocatableResource.Memory = rQuant.Value()
        case v1.ResourceNvidiaGPU:
            n.allocatableResource.NvidiaGPU = rQuant.Value()
        case v1.ResourcePods:
            n.allowedPodNumber = int(rQuant.Value())
        case v1.ResourceStorageScratch:
            n.allocatableResource.StorageScratch = rQuant.Value()
        case v1.ResourceStorageOverlay:
            n.allocatableResource.StorageOverlay = rQuant.Value()
        default:
            if v1helper.IsOpaqueIntResourceName(rName) {
                n.allocatableResource.SetOpaque(rName, rQuant.Value())
            }
        }
    }
    n.taints = node.Spec.Taints
    for i := range node.Status.Conditions {
        cond := &node.Status.Conditions[i]
        switch cond.Type {
        case v1.NodeMemoryPressure:
            n.memoryPressureCondition = cond.Status
        case v1.NodeDiskPressure:
            n.diskPressureCondition = cond.Status
        case v1.NodeDiskMountPressure:
            n.diskMountPressureCondition = cond.Status
        default:
            // We ignore other conditions.
        }
    }
    n.generation++
    return nil
}

当然处理缓存节点状态cache还缓存pod状态plugin/pkg/scheduler/schedulercache/cache.go

    podStates map[string]*podState
    nodes     map[string]*NodeInfo

说完了cache的机制后，我们回到上面，一个pod过来怎么样调度去bind呢。当一个pod需要被调度时候回执行
plugin/pkg/scheduler/core/generic_scheduler.go里面的Schedule

    err = g.cache.UpdateNodeNameToInfoMap(g.cachedNodeInfoMap)
    if err != nil {
        return "", err
    }

    trace.Step("Computing predicates")
    filteredNodes, failedPredicateMap, err := findNodesThatFit(pod, g.cachedNodeInfoMap, nodes, g.predicates, g.extenders, g.predicateMetaProducer, g.equivalenceCache)

这个里面有两个方法需要分析，第一是UpdateNodeNameToInfoMap它是更新被调度节点信息的，plugin/pkg/scheduler/schedulercache/cache.go

func (cache *schedulerCache) UpdateNodeNameToInfoMap(nodeNameToInfo map[string]*NodeInfo) error {
    cache.mu.Lock()
    defer cache.mu.Unlock()
    for name, info := range cache.nodes {
        if current, ok := nodeNameToInfo[name]; !ok || current.generation != info.generation {
            nodeNameToInfo[name] = info.Clone()
        }
    }
    for name := range nodeNameToInfo {
        if _, ok := cache.nodes[name]; !ok {
            delete(nodeNameToInfo, name)
        }
    }
    return nil
}

这个里面通过info.Clone()吧cache里面节点信息，复制给nodeNameToInfo这个调度时候使用的筛选节点。第二个要说的方法是重点，findNodesThatFit，这个是具体调度时候使用的，进入这个方法

fits, failedPredicates, err := podFitsOnNode(pod, meta, nodeNameToInfo[nodeName], predicateFuncs, ecache)

这个里面通过16协程并行去执行新节点检查，调用上面的podFitsOnNode方法去判读是否合适，而podFitsOnNode这个方法里面会遍历去判断筛选方法

    for predicateKey, predicate := range predicateFuncs {
        // If equivalenceCache is available
        if eCacheAvailable {
            // PredicateWithECache will returns it's cached predicate results
            fit, reasons, invalid = ecache.PredicateWithECache(pod, info.Node().GetName(), predicateKey, equivalenceHash)
        }

        if !eCacheAvailable || invalid {
            // we need to execute predicate functions since equivalence cache does not work
            fit, reasons, err = predicate(pod, meta, info)
            if err != nil {
                return false, []algorithm.PredicateFailureReason{}, err
            }

            if eCacheAvailable {
                // update equivalence cache with newly computed fit & reasons
                // TODO(resouer) should we do this in another thread? any race?
                ecache.UpdateCachedPredicateItem(pod, info.Node().GetName(), predicateKey, fit, reasons, equivalenceHash)
            }
        }

        if !fit {
            // eCache is available and valid, and predicates result is unfit, record the fail reasons
            failedPredicates = append(failedPredicates, reasons...)
        }
    }

通过for循环逐一执行predicate这个函数指针。这个函数就是之前注册的筛选函数，通过返回fit是否为ture来判断是和合适，如果不合适，通过第二参数reasons去获取失败的原因。看一个最简单的磁盘压力的筛选函数

func CheckNodeDiskPressurePredicate(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) {
    // is node under presure?
    if nodeInfo.DiskPressureCondition() == v1.ConditionTrue {
        return false, []algorithm.PredicateFailureReason{ErrNodeUnderDiskPressure}, nil
    }
    return true, nil, nil
}

就是上面说的返回fit是否合适和reason。好了，整个筛选的流程走完了！