client-go多集群管理

背景

通常在多云环境中或者多k8s集群中，一个与k8s相关管理服务需要同事管理多个k8s集群，有很多种方法可以去实现该功能，这里主要介绍一种通过一个config文件在配合client-go代码来实现多集群client的创建，client-go中的example中都是单集群的client的实现。

实现代码

通过查看client-go创建client的内部代码后，整理出如下创建多集群的代码，我得式列代码中包含了 argo、volcano、k8s原生client

import (
	"context"
	"log"
	"os"
	"path/filepath"

	"github.com/argoproj/argo-workflows/v3/cmd/argo/commands/client"
	argoapi "github.com/argoproj/argo-workflows/v3/pkg/apiclient"
	argo "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/util/homedir"
	"k8s.io/klog"
	"volcano.sh/apis/pkg/client/clientset/versioned"
)

type Clientset struct {
	Ctx           context.Context
	K8s           *kubernetes.Clientset
	Argo          *argo.Clientset
	ArgoAPIClient argoapi.Client
	Volcano       *versioned.Clientset
}

func NewClient(kubeconfig string) (*Clientset, error) {
	config, err := buildConfig(kubeconfig)
	if err != nil {
		return nil, err
	}
	argoClient, err := argo.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	k8sClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}
	// dail kubernetes cluster api server
	version, err := k8sClient.DiscoveryClient.ServerVersion()
	if err != nil {
		return nil, err
	}
	klog.Infof("connected kubernetes cluster, and version is: %s", version.String())
	// init argo api client
	ctx, apiClient := client.NewAPIClient()
	// init volcano client
	volcano, err := versioned.NewForConfig(config)
	if err != nil {
		return nil, err
	}
	return &Clientset{ctx, k8sClient, argoClient, apiClient, volcano}, nil
}

func NewClients(kubeconfig string) (map[string]*Clientset, error) {
	if kubeconfig == "" {
		kubeconfig = os.Getenv("KUBECONFIG")
	}

	if kubeconfig == "" && os.Getenv("KUBECONFIG") == "" {
		if home := homedir.HomeDir(); home != "" {
			kubeconfig = filepath.Join(home, ".kube", "config")
		}
	}

	f, err := os.ReadFile(kubeconfig)
	if err != nil {
		return nil, err
	}
	cfg, err := clientcmd.Load(f)
	if err != nil {
		return nil, err
	}

	clients := map[string]*Clientset{}
	for ctxName, ctx := range cfg.Contexts {
		clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
			&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig},
			&clientcmd.ConfigOverrides{
				AuthInfo:       *cfg.AuthInfos[ctx.AuthInfo],
				ClusterInfo:    *cfg.Clusters[ctx.Cluster],
				Context:        *ctx,
				CurrentContext: ctxName,
				Timeout:        "5s",
			})
		restCofnig, err := clientConfig.ClientConfig()
		if err != nil {
			return nil, err
		}

		k8sClient, err := kubernetes.NewForConfig(restCofnig)
		if err != nil {
			return nil, err
		}
		// 这里一定要调用Discovery().ServerVersion()，探测Kube Apiserver是否可用，因为kubernetes.NewForConfig(restCofnig)不会去检查服务是否可用，当服务不可用时，该方法不会返回错误的
		version, err := k8sClient.Discovery().ServerVersion()
		if err != nil {
			log.Printf("connect cluster %v failed: %v\n", ctx.Cluster, err.Error())
			return nil, err
		}
		klog.Infof("connected kubernetes cluster %s succeed, and version is: %s", ctx.Cluster, version.String())
		argoClient, err := argo.NewForConfig(restCofnig)
		if err != nil {
			return nil, err
		}

		c, apiClient, err := argoapi.NewClientFromOpts(argoapi.Opts{ClientConfig: clientConfig})
		if err != nil {
			return nil, err
		}

		volcano, err := versioned.NewForConfig(restCofnig)
		if err != nil {
			return nil, err
		}
		clients[ctx.Cluster] = &Clientset{c, k8sClient, argoClient, apiClient, volcano}
	}
	return clients, nil
}