/**
*Copy right 2008-2021 by Jacky Zong.
*All rights reserved.
*曾经沧海难为水,
*除却巫山不是雲。
*/
package route
import (
"context"
"fmt"
"k8s.io/client-go/util/workqueue"
queue "k8s.io/client-go/util/workqueue"
"k8s.io/cloud-provider-alibaba-cloud/cloud-controller-manager/utils"
"k8s.io/cloud-provider-alibaba-cloud/cloud-controller-manager/utils/metric"
"k8s.io/klog"
"net"
"reflect"
"time"
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
coreinformers "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
//v1node "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/cloud-provider"
"k8s.io/cloud-provider/node/helpers"
metrics "k8s.io/component-base/metrics/prometheus/ratelimiter"
controller "k8s.io/kube-aggregator/pkg/controllers"
nodeutil "k8s.io/kubernetes/pkg/util/node"
)
const (
// ROUTE_CONTROLLER route controller name
ROUTE_CONTROLLER = "route-controller"
// Maximum number of retries of node status update.
updateNodeStatusMaxRetries int = 3
)
// Routes is an abstract, pluggable interface for advanced routing rules.
type Routes interface {
// RouteTables get all available route tables.
RouteTables(ctx context.Context, clusterName string) ([]string, error)
// ListRoutes lists all managed routes that belong to the specified clusterName
ListRoutes(ctx context.Context, clusterName string, table string) ([]*cloudprovider.Route, error)
// CreateRoute creates the described managed route
// route.Name will be ignored, although the cloud-provider may use nameHint
// to create a more user-meaningful name.
CreateRoute(ctx context.Context, clusterName string, nameHint string, table string, route *cloudprovider.Route) error
// DeleteRoute deletes the specified managed route
// Route should be as returned by ListRoutes
DeleteRoute(ctx context.Context, clusterName string, table string, route *cloudprovider.Route) error
}
// RouteController response for route reconcile
type RouteController struct {
routes Routes
kubeClient clientset.Interface
clusterName string
clusterCIDR *net.IPNet
nodeLister corelisters.NodeLister
nodeListerSynced cache.InformerSynced
broadcaster record.EventBroadcaster
recorder record.EventRecorder
// Package workqueue provides a simple queue that supports the following
// features:
// * Fair: items processed in the order in which they are added.
// * Stingy: a single item will not be processed multiple times concurrently,
// and if an item is added multiple times before it can be processed, it
// will only be processed once.
// * Multiple consumers and producers. In particular, it is allowed for an
// item to be reenqueued while it is being processed.
// * Shutdown notifications.
queues map[string]queue.DelayingInterface
}
const NODE_QUEUE = "node.queue"
// New new route controller
func New(routes Routes,
kubeClient clientset.Interface,
nodeInformer coreinformers.NodeInformer,
clusterName string, clusterCIDR *net.IPNet) (*RouteController, error) {
if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil {
err := metrics.RegisterMetricAndTrackRateLimiterUsage(
ROUTE_CONTROLLER,
kubeClient.CoreV1().RESTClient().GetRateLimiter(),
)
if err != nil {
klog.Warningf("metrics initialized fail. %s", err.Error())
}
}
if clusterCIDR == nil {
return nil, fmt.Errorf("RouteController: Must specify clusterCIDR")
}
eventer, caster := broadcaster()
rc := &RouteController{
routes: routes,
kubeClient: kubeClient,
clusterName: clusterName,
clusterCIDR: clusterCIDR,
nodeLister: nodeInformer.Lister(),
nodeListerSynced: nodeInformer.Informer().HasSynced,
broadcaster: caster,
recorder: eventer,
queues: map[string]queue.DelayingInterface{
NODE_QUEUE: workqueue.NewNamedDelayingQueue(NODE_QUEUE),
},
}
rc.HandlerForNodeDeletion(
rc.queues[NODE_QUEUE],
nodeInformer.Informer(),
)
return rc, nil
}
func (rc *RouteController) HandlerForNodeDeletion(
que queue.DelayingInterface,
informer cache.SharedIndexInformer,
) {
informer.AddEventHandler(
cache.ResourceEventHandlerFuncs{
DeleteFunc: func(nodec interface{}) {
node, ok := nodec.(*v1.Node)
if !ok {
klog.Infof("not node type: %s\n", reflect.TypeOf(nodec))
return
}
if utils.IsExcludedNode(node) {
klog.Infof("ignore node with exclude node label %s", node.Name)
return
}
que.Add(node)
klog.Infof("node deletion event: %s, %s", node.Name, node.Spec.ProviderID)
},
},
)
}
// Run start route controller
func (rc *RouteController) Run(stopCh <-chan struct{}, syncPeriod time.Duration) {
defer utilruntime.HandleCrash()
klog.Info("starting route controller")
defer klog.Info("shutting down route controller")
if !controller.WaitForCacheSync(ROUTE_CONTROLLER, stopCh, rc.nodeListerSynced) {
return
}
if rc.broadcaster != nil {
sink := &v1core.EventSinkImpl{
Interface: v1core.New(rc.kubeClient.CoreV1().RESTClient()).Events(""),
}
rc.broadcaster.StartRecordingToSink(sink)
}
// TODO: If we do just the full Resync every 5 minutes (default value)
// that means that we may wait up to 5 minutes before even starting
// creating a route for it. This is bad.
// We should have a watch on node and if we observe a new node (with CIDR?)
// trigger reconciliation for that node.
go wait.NonSlidingUntil(func() {
if err := rc.reconcile(); err != nil {
klog.Errorf("Couldn't reconcile node routes: %v", err)
}
}, syncPeriod, stopCh)
go wait.Until(
func() {
que := rc.queues[NODE_QUEUE]
for {
func() {
// Workerqueue ensures that a single key would not be process
// by two worker concurrently, so multiple workers is safe here.
key, quit := que.Get()
if quit {
return
}
defer que.Done(key)
node, ok := key.(*v1.Node)
if !ok {
klog.Errorf("not type of *v1.Node, %s", reflect.TypeOf(key))
return
}
klog.Infof("worker: queued sync for [%s] node deletion with route", node.Name)
start := time.Now()
if err := rc.syncd(node); err != nil {
que.AddAfter(key, 2*time.Minute)
klog.Errorf("requeue: sync route for node %s, error %v", node.Name, err)
}
metric.RouteLatency.WithLabelValues("delete").Observe(metric.MsSince(start))
}()
}
},
2*time.Second,
stopCh,
)
<-stopCh
}
func (rc *RouteController) syncd(node *v1.Node) error {
if utils.IsExcludedNode(node) {
return nil
}
if node.Spec.PodCIDR == "" {
klog.Warningf("Node %s PodCIDR is nil, skip delete route", node.Name)
return nil
}
if node.Spec.ProviderID == "" {
klog.Warningf("Node %s has no Provider ID, skip delete route", node.Name)
return nil
}
ctx := context.Background()
tabs, err := rc.routes.RouteTables(ctx, rc.clusterName)
if err != nil {
return fmt.Errorf("RouteTables: %s", err.Error())
}
for _, table := range tabs {
route := &cloudprovider.Route{
Name: node.Spec.ProviderID,
TargetNode: types.NodeName(node.Spec.ProviderID),
DestinationCIDR: node.Spec.PodCIDR,
}
if err := rc.routes.DeleteRoute(
ctx, rc.clusterName, table, route,
); err != nil {
klog.Errorf(
"delete route %s %s from table %s, %s", route.Name, route.DestinationCIDR, table, err.Error())
return fmt.Errorf("node deletion, delete route error: %s", err.Error())
}
klog.Infof("node deletion: delete route %s %s from table %s SUCCESS.", route.Name, route.DestinationCIDR, table)
}
return nil
}
func (rc *RouteController) reconcile() error {
ctx := context.Background()
start := time.Now()
nodes, err := rc.nodeLister.List(labels.Everything())
if err != nil {
return fmt.Errorf("error listing nodes: %v", err)
}
tabs, err := rc.routes.RouteTables(ctx, rc.clusterName)
if err != nil {
return fmt.Errorf("RouteTables: %s", err.Error())
}
for _, table := range tabs {
//ListRoutes & Sync
routeList, err := rc.routes.ListRoutes(ctx, rc.clusterName, table)
if err != nil {
return fmt.Errorf("error listing routes: %v", err)
}
if err := rc.sync(ctx, table, nodes, routeList); err != nil {
return fmt.Errorf("reconcile route for table [%s] error: %s", table, err.Error())
}
}
metric.RouteLatency.WithLabelValues("reconcile").Observe(metric.MsSince(start))
return nil
}
// Aoxn: Alibaba cloud does not support concurrent route operation
func (rc *RouteController) sync(ctx context.Context, table string, nodes []*v1.Node, routes []*cloudprovider.Route) error {
//try delete conflicted route from vpc route table.
for _, route := range routes {
if !rc.isResponsibleForRoute(route) {
continue
}
// Check if this route is a blackhole, or applies to a node we know about & has an incorrect CIDR.
if route.Blackhole || rc.isRouteConflicted(nodes, route) {
// Aoxn: Alibaba cloud does not support concurrent route operation
klog.Infof("Deleting route %s %s", route.Name, route.DestinationCIDR)
if err := rc.routes.DeleteRoute(ctx, rc.clusterName, table, route); err != nil {
klog.Errorf("Could not delete route %s %s from table %s, %s", route.Name, route.DestinationCIDR, table, err.Error())
continue
}
klog.Infof("Delete route %s %s from table %s SUCCESS.", route.Name, route.DestinationCIDR, table)
}
}
cached := RouteCacheMap(routes)