client-go批量删除异常pod与重启次数过多pod

client-go批量删除异常pod与重启次数过多pod

• 当K8S集群内生成大量被驱逐的pod时，在master负载较高时使用kubectl批量删除异常pod速度很慢，故使用原生的client-go库优化删除速度
• 批量删除pod方式：./gokit -m 虚拟机实测删除3000个pod大约耗时18s
• 单个删除pod方式：./gokit -s
• 删除重启次数大于xx的pod：./gokit -r -c 10

package main

import (
	"context"
	"fmt"
	"log"

	"github.com/spf13/pflag"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	// 后台删除，不夯前台，提升删除速度
	deletePolicy metav1.DeletionPropagation = "Background"
	// 立即删除
	gracetime     int64                = 0
	deleteOptions metav1.DeleteOptions = metav1.DeleteOptions{
		PropagationPolicy:  &deletePolicy,
		GracePeriodSeconds: &gracetime,
	}
)

// 初始化client
func initClientSet() *kubernetes.Clientset {
	// 获取reset config
	config, err := clientcmd.BuildConfigFromFlags("", "/root/.kube/config")
	if err != nil {
		log.Fatal(err.Error())
	}

	// 实例clienetset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		log.Fatal(err.Error())
	}

	return clientset
}

// 获取所有异常POD
func singleDeletePods(clientset *kubernetes.Clientset) {
	podlist, _ := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{
		FieldSelector: "status.phase!=Running,status.phase!=Succeeded,status.phase!=Unknown",
	})

	for _, pod := range podlist.Items {
		err := clientset.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
		if err != nil {
			fmt.Printf("命名空间：%s--POD名称：%s 删除失败!\n", pod.Namespace, pod.Name)
		}
	}
}

// 批量删除异常pod
func multiDeleteErrorPods(clientset *kubernetes.Clientset) {
	// 获取所有namespaces,批量删除必须要指定namespace
	namespaceList, err := clientset.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		log.Fatal(err.Error())
	}

	// 删除所有pod
	for _, namespace := range namespaceList.Items {
		err = clientset.CoreV1().Pods(namespace.Name).DeleteCollection(context.TODO(), deleteOptions, metav1.ListOptions{
			FieldSelector: "status.phase!=Running,status.phase!=Succeeded,status.phase!=Unknown",
		})
		if err != nil {
			fmt.Println(namespace.Name + "下pods删除失败" + err.Error())
		}
	}
}

// 删除重启次数过多pod,k8s目前不支持FieldSelector选择重启次数的，所以只能单个删除
func singleDeleteRestartPods(clientset *kubernetes.Clientset, count int32) {
	podlist, _ := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
	for _, pod := range podlist.Items {
		var restartCount int32
		for index, _ := range pod.Spec.Containers {
			restartCount += pod.Status.ContainerStatuses[index].RestartCount
		}
		if restartCount >= count {
			err := clientset.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, deleteOptions)
			if err != nil {
				fmt.Println(pod.Namespace + "-" + pod.Name + "删除失败")
			}
		}
	}
}

func main() {
	// 初始化flag
	var single = pflag.BoolP("single", "s", false, "删除异常POD(default false)")
	var multi = pflag.BoolP("multi", "m", false, "批量删除POD(default false)")
	var restart = pflag.BoolP("restart", "r", false, "删除重启次数大于xx的POD(default false)")
	var restartCount = pflag.Int32P("count", "c", 10, "过滤重启次数大于xx的POD(default 10)")

	pflag.Parse()

	// 初始化clientset
	clientset := initClientSet()
	if *multi {
		multiDeleteErrorPods(clientset)
	} else if *single {
		singleDeletePods(clientset)
	} else if *restart {
		singleDeleteRestartPods(clientset, *restartCount)
	} else {
		pflag.PrintDefaults()
	}
}