自定义admission controller(Mutating) supports kubernetes 1.24.6

本文使用admission的方式对开发环境的POD增加label

编写openssl.cnf

[ CA_default ]
copy_extensions = copy 
[req]
distinguished_name = req_distinguished_name
x509_extensions = v3_req
prompt = no
[req_distinguished_name]
# 国家
C = CN
# 省份
ST = Shenzhen
# 城市
L = Shenzhen
# 组织
O = Arvin
# 部门
OU = Arvin
# 域名
CN = label-namespace-env.kube-system.svc
[v3_req]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
# 解析域名
DNS.1 = label-namespace-env.kube-system.svc
# 可配置多个域名，如下
DNS.2 = label-namespace-env.kube-system

生成CA秘钥

# 生成私钥，密码可以输入123456, CA机构的 私钥
openssl genrsa -des3 -out ca.key 2048
# 使用私钥来签名证书  CA机构的根证书
openssl req -new -key ca.key -out ca.csr
# 使用私钥+证书来生成公钥  CA证书的公钥(一般在系统内部基础(window,linux))
openssl x509 -req -days 3650 -in ca.csr -signkey ca.key -out ca.crt

# 生成私钥，密码可以输入123456 ,服务器的私钥
openssl genpkey -algorithm RSA -out server/server.key
# 使用私钥来签名证书  ,服务器的根证书
openssl req -new -nodes -key server/server.key -out server/server.csr -config openssl.cnf -extensions 'v3_req'
# 使用CA颁发的证书来生成公钥 服务器的公钥
openssl x509 -req -days 3650 -in server/server.csr -out server/server.pem -CA ca.crt -CAkey ca.key -CAcreateserial -extfile ./openssl.cnf -extensions 'v3_req'
# csr格式的根证书转crt格式
openssl x509 -req -days 3650 -in server/server.csr -signkey server/server.key -out ./server/server.crt
#服务器 私钥key转pem格式
openssl rsa -in ./server/server.key -out ./server/key.pem
#创建加密秘钥,给后面Webhook的Deployment使用
kubectl create secret generic label-namespace-env-certs -n kube-system   --from-file=key.pem=server/key.pem   --from-file=cert.pem=server/server.pem

# 查看CA.CRT证书
cat ./ca.crt | base64 | tr -d '\n'

编写Webhook

这里使用 https://github.com/slok/kubewebhook 框架解析的kubernetes的API调用

package main

import (
	"context"
	"flag"
	"fmt"
	corev1 "k8s.io/api/core/v1"
	"net/http"
	"os"

	"github.com/sirupsen/logrus"
	kwhhttp "github.com/slok/kubewebhook/v2/pkg/http"
	kwhlogrus "github.com/slok/kubewebhook/v2/pkg/log/logrus"
	kwhmodel "github.com/slok/kubewebhook/v2/pkg/model"
	kwhmutating "github.com/slok/kubewebhook/v2/pkg/webhook/mutating"
	mutatingwh "github.com/slok/kubewebhook/v2/pkg/webhook/mutating"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type config struct {
	certFile string
	keyFile  string
}

func initFlags() *config {
	cfg := &config{}

	fl := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	fl.StringVar(&cfg.certFile, "tls-cert-file", "/etc/certs/cert.pem", "TLS certificate file")
	fl.StringVar(&cfg.keyFile, "tls-key-file", "/etc/certs/key.pem", "TLS key file")

	_ = fl.Parse(os.Args[1:])
	return cfg
}

// curl https://label-namespace-env.kube-system
func main() {
	logrusLogEntry := logrus.NewEntry(logrus.New())
	logrusLogEntry.Logger.SetLevel(logrus.DebugLevel)
	logger := kwhlogrus.NewLogrus(logrusLogEntry)
	cfg := initFlags()
	mt := kwhmutating.MutatorFunc(func(context context.Context, admission *kwhmodel.AdmissionReview, obj metav1.Object) (*kwhmutating.MutatorResult, error) {
		pod, ok := obj.(*corev1.Pod)
		if !ok {
			return &kwhmutating.MutatorResult{}, nil
		}
		fmt.Println("Admission", admission.Namespace)
		if admission.Namespace == "xxxx-dev" {
			// Mutate our object with the required annotations.
			fmt.Println("开发环境")
			if pod.Annotations == nil {
				pod.Annotations = make(map[string]string)
			}
			pod.Annotations["ENV"] = "开发环境"
		} else if admission.Namespace == "xxxx-test" {
			fmt.Println("测试环境")
			if pod.Annotations == nil {
				pod.Annotations = make(map[string]string)
			}
			pod.Annotations["ENV"] = "测试环境"
		}
		fmt.Println("AFTER:", pod)
		return &kwhmutating.MutatorResult{MutatedObject: pod}, nil
	})

	// We don't use any type, it works for any type.
	mcfg := mutatingwh.WebhookConfig{
		ID:      "labeler",
		Mutator: mt,
		Logger:  logger,
	}
	wh, err := mutatingwh.NewWebhook(mcfg)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook: %s", err)
		os.Exit(1)
	}

	// Get the handler for our webhook.
	whHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: wh, Logger: logger})
	if err != nil {
		fmt.Fprintf(os.Stderr, "error creating webhook handler: %s", err)
		os.Exit(1)
	}
	logger.Infof("Listening on :8080")
	err = http.ListenAndServeTLS(":8080", cfg.certFile, cfg.keyFile, whHandler)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error serving webhook: %s", err)
		os.Exit(1)
	}
}

简单编写Dockerfile打包应用

FROM golang:1.19
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone
WORKDIR /code
RUN go env -w GO111MODULE=on
RUN go env -w GOPROXY=https://goproxy.cn,https://goproxy.io,direct
WORKDIR /workdir
COPY . .
RUN go mod tidy
RUN go build -o server-api .
CMD ["/workdir/server-api"]

编写Deployment

kind: Deployment
apiVersion: apps/v1
metadata:
  name: label-namespace-env
  namespace: kube-system
spec:
  replicas: 3
  selector:
    matchLabels:
      k8s-app: label-namespace-env
  template:
    metadata:
      labels:
        k8s-app: label-namespace-env
    spec:
      volumes:
        - name: webhook-certs
          secret:
            secretName: label-namespace-env-certs
            defaultMode: 420
      containers:
        - name: webhook
          image: 'xxxxx/xxxx/label-namespace-env:v2'
          resources: {}
          volumeMounts:
            - name: webhook-certs
              mountPath: /etc/certs
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          imagePullPolicy: IfNotPresent

Service

kind: Service
apiVersion: v1
metadata:
  name: label-namespace-env
  namespace: kube-system
spec:
  ports:
    - name: webhook
      protocol: TCP
      port: 443
      targetPort: 8080
  selector:
    k8s-app: label-namespace-env
  type: ClusterIP

编写MutatingWebhookConfiguration

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: label-namespace-env
webhooks:
  - name: label-namespace-env.kube-system.svc #这里是webhook应用发布后的svc
    admissionReviewVersions: ["v1", "v1beta1"]
    sideEffects: None
    timeoutSeconds: 15
    #排除webhook自身
    namespaceSelector:
      matchExpressions:
      - key: remove
        operator: DoesNotExist
    #忽略错误影响 failurePolicy: Ignore
    clientConfig:
      service:
        name: label-namespace-env
        namespace: kube-system
        path: "/"
      caBundle:  #cat ./ca.crt | base64 | tr -d '\n' 命令输出的内容
    rules:
      - operations: [ "CREATE" ]
        apiGroups: [""]
        apiVersions: ["v1"]
        resources: ["pods"]
~