Alibaba Cloud 微服务 Route 源码-Java 语言-立哥开发

/**
*Copy right 2008-2021 by Jacky Zong.
*All rights reserved.
*曾经沧海难为水，
*除却巫山不是雲。

*/

package route

import (
    "context"
    "fmt"
    "k8s.io/client-go/util/workqueue"
    queue "k8s.io/client-go/util/workqueue"
    "k8s.io/cloud-provider-alibaba-cloud/cloud-controller-manager/utils"
    "k8s.io/cloud-provider-alibaba-cloud/cloud-controller-manager/utils/metric"
    "k8s.io/klog"
    "net"
    "reflect"
    "time"

    "strings"

    "k8s.io/api/core/v1"
    "k8s.io/apimachinery/pkg/api/errors"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/apimachinery/pkg/labels"
    "k8s.io/apimachinery/pkg/types"
    utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    "k8s.io/apimachinery/pkg/util/wait"
    coreinformers "k8s.io/client-go/informers/core/v1"
    clientset "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/kubernetes/scheme"
    v1core "k8s.io/client-go/kubernetes/typed/core/v1"
    corelisters "k8s.io/client-go/listers/core/v1"
    "k8s.io/client-go/tools/cache"
    "k8s.io/client-go/tools/record"
    //v1node "k8s.io/kubernetes/pkg/api/v1/node"
    "k8s.io/cloud-provider"
    "k8s.io/cloud-provider/node/helpers"
    metrics "k8s.io/component-base/metrics/prometheus/ratelimiter"
    controller "k8s.io/kube-aggregator/pkg/controllers"
    nodeutil "k8s.io/kubernetes/pkg/util/node"
)

const (
    // ROUTE_CONTROLLER route controller name
    ROUTE_CONTROLLER = "route-controller"

    // Maximum number of retries of node status update.
    updateNodeStatusMaxRetries int = 3
)

// Routes is an abstract, pluggable interface for advanced routing rules.
type Routes interface {
    // RouteTables get all available route tables.
    RouteTables(ctx context.Context, clusterName string) ([]string, error)
    // ListRoutes lists all managed routes that belong to the specified clusterName
    ListRoutes(ctx context.Context, clusterName string, table string) ([]*cloudprovider.Route, error)
    // CreateRoute creates the described managed route
    // route.Name will be ignored, although the cloud-provider may use nameHint
    // to create a more user-meaningful name.
    CreateRoute(ctx context.Context, clusterName string, nameHint string, table string, route *cloudprovider.Route) error
    // DeleteRoute deletes the specified managed route
    // Route should be as returned by ListRoutes
    DeleteRoute(ctx context.Context, clusterName string, table string, route *cloudprovider.Route) error
}

// RouteController response for route reconcile
type RouteController struct {
    routes           Routes
    kubeClient       clientset.Interface
    clusterName      string
    clusterCIDR      *net.IPNet
    nodeLister       corelisters.NodeLister
    nodeListerSynced cache.InformerSynced
    broadcaster      record.EventBroadcaster
    recorder         record.EventRecorder
    // Package workqueue provides a simple queue that supports the following
    // features:
    //  * Fair: items processed in the order in which they are added.
    //  * Stingy: a single item will not be processed multiple times concurrently,
    //      and if an item is added multiple times before it can be processed, it
    //      will only be processed once.
    //  * Multiple consumers and producers. In particular, it is allowed for an
    //      item to be reenqueued while it is being processed.
    //  * Shutdown notifications.
    queues map[string]queue.DelayingInterface
}

const NODE_QUEUE = "node.queue"

// New new route controller
func New(routes Routes,
    kubeClient clientset.Interface,
    nodeInformer coreinformers.NodeInformer,
    clusterName string, clusterCIDR *net.IPNet) (*RouteController, error) {

    if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil {
        err := metrics.RegisterMetricAndTrackRateLimiterUsage(
            ROUTE_CONTROLLER,
            kubeClient.CoreV1().RESTClient().GetRateLimiter(),
        )
        if err != nil {
            klog.Warningf("metrics initialized fail. %s", err.Error())
        }
    }

    if clusterCIDR == nil {
        return nil, fmt.Errorf("RouteController: Must specify clusterCIDR")
    }

    eventer, caster := broadcaster()

    rc := &RouteController{
        routes:           routes,
        kubeClient:       kubeClient,
        clusterName:      clusterName,
        clusterCIDR:      clusterCIDR,
        nodeLister:       nodeInformer.Lister(),
        nodeListerSynced: nodeInformer.Informer().HasSynced,
        broadcaster:      caster,
        recorder:         eventer,
        queues: map[string]queue.DelayingInterface{
            NODE_QUEUE: workqueue.NewNamedDelayingQueue(NODE_QUEUE),
        },
    }

    rc.HandlerForNodeDeletion(
        rc.queues[NODE_QUEUE],
        nodeInformer.Informer(),
    )

    return rc, nil
}

func (rc *RouteController) HandlerForNodeDeletion(
    que queue.DelayingInterface,
    informer cache.SharedIndexInformer,
) {
    informer.AddEventHandler(
        cache.ResourceEventHandlerFuncs{
            DeleteFunc: func(nodec interface{}) {
                node, ok := nodec.(*v1.Node)
                if !ok {
                    klog.Infof("not node type: %s\n", reflect.TypeOf(nodec))
                    return
                }
                if utils.IsExcludedNode(node) {
                    klog.Infof("ignore node with exclude node label %s", node.Name)
                    return
                }
                que.Add(node)
                klog.Infof("node deletion event: %s, %s", node.Name, node.Spec.ProviderID)
            },
        },
    )
}

// Run start route controller
func (rc *RouteController) Run(stopCh <-chan struct{}, syncPeriod time.Duration) {
    defer utilruntime.HandleCrash()

    klog.Info("starting route controller")
    defer klog.Info("shutting down route controller")

    if !controller.WaitForCacheSync(ROUTE_CONTROLLER, stopCh, rc.nodeListerSynced) {
        return
    }

    if rc.broadcaster != nil {
        sink := &v1core.EventSinkImpl{
            Interface: v1core.New(rc.kubeClient.CoreV1().RESTClient()).Events(""),
        }
        rc.broadcaster.StartRecordingToSink(sink)
    }

    // TODO: If we do just the full Resync every 5 minutes (default value)
    // that means that we may wait up to 5 minutes before even starting
    // creating a route for it. This is bad.
    // We should have a watch on node and if we observe a new node (with CIDR?)
    // trigger reconciliation for that node.
    go wait.NonSlidingUntil(func() {
        if err := rc.reconcile(); err != nil {
            klog.Errorf("Couldn't reconcile node routes: %v", err)
        }
    }, syncPeriod, stopCh)

    go wait.Until(
        func() {
            que := rc.queues[NODE_QUEUE]
            for {
                func() {
                    // Workerqueue ensures that a single key would not be process
                    // by two worker concurrently, so multiple workers is safe here.
                    key, quit := que.Get()
                    if quit {
                        return
                    }
                    defer que.Done(key)
                    node, ok := key.(*v1.Node)
                    if !ok {
                        klog.Errorf("not type of *v1.Node, %s", reflect.TypeOf(key))
                        return
                    }
                    klog.Infof("worker: queued sync for [%s] node deletion with route", node.Name)
                    start := time.Now()
                    if err := rc.syncd(node); err != nil {
                        que.AddAfter(key, 2*time.Minute)
                        klog.Errorf("requeue: sync route for node %s, error %v", node.Name, err)
                    }
                    metric.RouteLatency.WithLabelValues("delete").Observe(metric.MsSince(start))
                }()
            }
        },
        2*time.Second,
        stopCh,
    )
    <-stopCh
}

func (rc *RouteController) syncd(node *v1.Node) error {
    if utils.IsExcludedNode(node) {
        return nil
    }
    if node.Spec.PodCIDR == "" {
        klog.Warningf("Node %s PodCIDR is nil, skip delete route", node.Name)
        return nil
    }
    if node.Spec.ProviderID == "" {
        klog.Warningf("Node %s has no Provider ID, skip delete route", node.Name)
        return nil
    }

    ctx := context.Background()
    tabs, err := rc.routes.RouteTables(ctx, rc.clusterName)
    if err != nil {
        return fmt.Errorf("RouteTables: %s", err.Error())
    }
    for _, table := range tabs {
        route := &cloudprovider.Route{
            Name:            node.Spec.ProviderID,
            TargetNode:      types.NodeName(node.Spec.ProviderID),
            DestinationCIDR: node.Spec.PodCIDR,
        }
        if err := rc.routes.DeleteRoute(
            ctx, rc.clusterName, table, route,
        ); err != nil {
            klog.Errorf(
                "delete route %s %s from table %s, %s", route.Name, route.DestinationCIDR, table, err.Error())
            return fmt.Errorf("node deletion, delete route error: %s", err.Error())
        }
        klog.Infof("node deletion: delete route %s %s from table %s SUCCESS.", route.Name, route.DestinationCIDR, table)
    }
    return nil
}

func (rc *RouteController) reconcile() error {
    ctx := context.Background()
    start := time.Now()
    nodes, err := rc.nodeLister.List(labels.Everything())
    if err != nil {
        return fmt.Errorf("error listing nodes: %v", err)
    }
    tabs, err := rc.routes.RouteTables(ctx, rc.clusterName)
    if err != nil {
        return fmt.Errorf("RouteTables: %s", err.Error())
    }
    for _, table := range tabs {
        //ListRoutes & Sync
        routeList, err := rc.routes.ListRoutes(ctx, rc.clusterName, table)
        if err != nil {
            return fmt.Errorf("error listing routes: %v", err)
        }
        if err := rc.sync(ctx, table, nodes, routeList); err != nil {
            return fmt.Errorf("reconcile route for table [%s] error: %s", table, err.Error())
        }
    }
    metric.RouteLatency.WithLabelValues("reconcile").Observe(metric.MsSince(start))
    return nil
}

// Aoxn: Alibaba cloud does not support concurrent route operation
func (rc *RouteController) sync(ctx context.Context, table string, nodes []*v1.Node, routes []*cloudprovider.Route) error {

    //try delete conflicted route from vpc route table.
    for _, route := range routes {
        if !rc.isResponsibleForRoute(route) {
            continue
        }

        // Check if this route is a blackhole, or applies to a node we know about & has an incorrect CIDR.
        if route.Blackhole || rc.isRouteConflicted(nodes, route) {

            // Aoxn: Alibaba cloud does not support concurrent route operation
            klog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR)
            if err := rc.routes.DeleteRoute(ctx, rc.clusterName, table, route); err != nil {
                klog.Errorf("Could not delete route %s %s from table %s, %s", route.Name, route.DestinationCIDR, table, err.Error())
                continue
            }
            klog.Infof("Delete route %s %s from table %s SUCCESS.", route.Name, route.DestinationCIDR, table)
        }
    }
    cached := RouteCacheMap(routes)