开发apiserver实现一个容器镜像白名单的 K8S 准入控制器

我们使用go编写validate和mutate

package main

import (
	"context"
	"crypto/tls"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"webhook/pkg"

	"k8s.io/klog/v2"
)

func main() {
	var param pkg.WhSvrParam
	flag.IntVar(&param.Port, "port", 443, "webhook server port")
	flag.StringVar(&param.Certfile, "tlsCertfile", "/etc/webhook/certs/tls.crt", "x509 certification file")
	flag.StringVar(&param.KeyFile, "tlsKeyfile", "/etc/webhook/certs/tls.key", "x509 private key file")
	flag.Parse()

	certficate, err := tls.LoadX509KeyPair(param.Certfile, param.KeyFile)
	if err != nil {
		klog.Errorf("Failed to load key pair: &v", err)
		return
	}

	whsrv := pkg.WebhookServer{
		Server: &http.Server{
			Addr: fmt.Sprintf(":%d", param.Port),
			TLSConfig: &tls.Config{
				Certificates: []tls.Certificate{certficate},
			},
		},
		WhiteListRegistries: strings.Split(os.Getenv("WHITELIST_REGISTRIES"), ","),
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/validate", whsrv.Handler)
	mux.HandleFunc("/mutate", whsrv.Handler)
	whsrv.Server.Handler = mux

	go func() {
		if err := whsrv.Server.ListenAndServeTLS("", ""); err != nil {
			klog.Errorf("failed to listen and serve webhook: %v", err)
		}
	}()

	klog.Info("Serevr start")

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
	<-signalChan

	klog.Info("get os shuting...")
	if err := whsrv.Server.Shutdown(context.Background()); err != nil {
		klog.Errorf("http server shutdown err: %v", err)
	}
}

package pkg

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	admissionv1 "k8s.io/api/admission/v1"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/klog/v2"
)

type patchOperation struct {
	Op    string      `json:"op"`
	Path  string      `json:"path"`
	Value interface{} `json:"value,omitempty"`
}

const (
	AnnotationMutateKey = "io.ydzs.admission-registry/mutate"
	AnnotationStatusKey = "io.ydzs.admission-registry/status"
)

var (
	runtimeScheme = runtime.NewScheme()
	codeFactory   = serializer.NewCodecFactory(runtimeScheme)
	deserializer  = codeFactory.UniversalDeserializer()
)

type WhSvrParam struct {
	Port     int
	Certfile string
	KeyFile  string
}

type WebhookServer struct {
	Server              *http.Server
	WhiteListRegistries []string
}

func (s *WebhookServer) Handler(w http.ResponseWriter, r *http.Request) {
	var body []byte
	if r.Body != nil {
		if data, err := ioutil.ReadAll(r.Body); err == nil {
			body = data
		}
	}

	if len(body) == 0 {
		klog.Errorf("empty data body")
		http.Error(w, "empty data body", http.StatusBadRequest)
		return
	}

	contentType := r.Header.Get("Content-Type")
	if contentType != "application/json" {
		klog.Errorf("content-type id %s, but expect application/json", contentType)
		http.Error(w, "content-type invalid, expect application/json", http.StatusBadRequest)
		return
	}

	var admissionReponse *admissionv1.AdmissionResponse
	requestedAdmissionReview := admissionv1.AdmissionReview{}

	if _, _, err := deserializer.Decode(body, nil, &requestedAdmissionReview); err != nil {
		klog.Errorf("can not decode body: %v", err)
		admissionReponse = &admissionv1.AdmissionResponse{
			Result: &metav1.Status{
				Code:    http.StatusInternalServerError,
				Message: err.Error(),
			},
		}
	} else {
		if r.URL.Path == "/mutate" {
			admissionReponse = s.mutate(&requestedAdmissionReview)
		} else if r.URL.Path == "/validate" {
			admissionReponse = s.validate(&requestedAdmissionReview)
		}
	}

	//构造返回的 admissionreview
	reponseAdmissionReview := admissionv1.AdmissionReview{}
	reponseAdmissionReview.APIVersion = requestedAdmissionReview.APIVersion
	reponseAdmissionReview.Kind = requestedAdmissionReview.Kind

	if admissionReponse != nil {
		reponseAdmissionReview.Response = admissionReponse
		if requestedAdmissionReview.Request != nil {
			reponseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID
		}
	}

	klog.Info(fmt.Printf("sending reponse: %v", requestedAdmissionReview.Response))

	respBytes, err := json.Marshal(reponseAdmissionReview)
	if err != nil {
		klog.Errorf("can not encode reponse: %v ", err)
		http.Error(w, fmt.Sprintf("could not encode reponse: %v", err), http.StatusInternalServerError)
	}

	klog.Info("read to write reponse...")
	if _, err := w.Write(respBytes); err != nil {
		klog.Errorf("can not write response: %v", err)
		http.Error(w, fmt.Sprintf("could not write reponse: %v", err), http.StatusInternalServerError)
	}

}

// validate pod
func (serv *WebhookServer) validate(ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
	req := ar.Request
	var (
		allowed = true
		code    = 200
		message = ""
	)

	klog.Infof("AdmissionReview for Kind=%s, Namespace=%s Name=%v UID=%v Operation=%v UserInfo=%v",
		req.Kind.Kind, req.Namespace, req.Name, req.UID, req.Operation, req.UserInfo)

	var pod corev1.Pod
	if err := json.Unmarshal(req.Object.Raw, &pod); err != nil {
		klog.Errorf("Could not unmarshal raw object: %v", err)
		allowed = false
		code = 400
		return &admissionv1.AdmissionResponse{
			Allowed: allowed,
			Result: &metav1.Status{
				Code:    int32(code),
				Message: err.Error(),
			},
		}
	}
	for _, container := range pod.Spec.Containers {
		var whitelisted = false
		for _, reg := range serv.WhiteListRegistries {
			if strings.HasPrefix(container.Image, reg) {
				whitelisted = true
			}
		}
		if !whitelisted {
			allowed = false
			code = 403
			message = fmt.Sprintf("%s image comes from an untrusted registry! Only images from %v are allowed.",
				container.Image, serv.WhiteListRegistries)
			break
		}
	}

	return &admissionv1.AdmissionResponse{
		Allowed: allowed,
		Result: &metav1.Status{
			Code:    int32(code),
			Message: message,
		},
	}
}

func (s *WebhookServer) mutate(ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
	req := ar.Request

	var (
		objectMeta *metav1.ObjectMeta
	)

	klog.Infof("admissionreview for kind=%s, namespace=%s", req.Kind.Kind, req.Namespace)

	switch req.Kind.Kind {
	case "Deployment":
		var deployment appsv1.Deployment
		if err := json.Unmarshal(req.Object.Raw, &deployment); err != nil {
			klog.Errorf("can not unmarshal %v", err)
			return &admissionv1.AdmissionResponse{
				Result: &metav1.Status{
					Code:    http.StatusBadRequest,
					Message: err.Error(),
				},
			}
		}
		objectMeta = &deployment.ObjectMeta
	case "Service":
		var service corev1.Service
		if err := json.Unmarshal(req.Object.Raw, &service); err != nil {
			klog.Errorf("can not unmarshal %v", err)
			return &admissionv1.AdmissionResponse{
				Result: &metav1.Status{
					Code:    http.StatusBadRequest,
					Message: err.Error(),
				},
			}
		}
		objectMeta = &service.ObjectMeta
	default:
		return &admissionv1.AdmissionResponse{
			Result: &metav1.Status{
				Code:    http.StatusBadRequest,
				Message: fmt.Sprintf("can not handle the kind %s object", req.Kind.Kind),
			},
		}
	}

	if !mutateRequired(objectMeta) {
		return &admissionv1.AdmissionResponse{
			Allowed: true,
		}
	}

	annotations := map[string]string{AnnotationStatusKey: "mutated"}

	var patch []patchOperation

	patch = append(patch, mutateAnnotations(objectMeta.GetAnnotations(), annotations)...)

	fmt.Println("********")
	fmt.Println(patch)
	fmt.Println("********")
	patchBytes, err := json.Marshal(patch)
	if err != nil {
		klog.Errorf("patch marshal error: %v", err)
		return &admissionv1.AdmissionResponse{
			Result: &metav1.Status{
				Code:    http.StatusBadRequest,
				Message: err.Error(),
			},
		}
	}

	return &admissionv1.AdmissionResponse{
		Allowed: true,
		Patch:   patchBytes,
		PatchType: func() *admissionv1.PatchType {
			pt := admissionv1.PatchTypeJSONPatch
			return &pt
		}(),
	}
}

func mutateAnnotations(target map[string]string, added map[string]string) (patch []patchOperation) {
	for key, value := range added {
		if target == nil || target[key] == "" {
			target = map[string]string{}
			patch = append(patch, patchOperation{
				Op:   "add",
				Path: "/metadata/annotations",
				Value: map[string]string{
					key: value,
				},
			})
		} else {
			patch = append(patch, patchOperation{
				Op:    "replace",
				Path:  "/metadata/annotations/" + key,
				Value: value,
			})
		}
	}
	return
}

func mutateRequired(metadata *metav1.ObjectMeta) bool {
	annotations := metadata.GetAnnotations()

	if annotations == nil {
		annotations = map[string]string{}
	}

	var required bool

	switch strings.ToLower(annotations[AnnotationMutateKey]) {
	case "n", "no", "false", "off":
		required = false
	default:
		required = true
	}

	status := annotations[AnnotationStatusKey]

	if strings.ToLower(status) == "mutated" {
		required = false
	}

	klog.Infof("mutation policy for %s/%s: required: %v", metadata.Name, metadata.Namespace, required)
	return required
}

dockerfile如下

FROM golang:1.18 as builder

RUN apt-get -y update && apt-get -y install upx

WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum

# Copy the go source
COPY main.go main.go
COPY pkg/ pkg/

# Build
ENV CGO_ENABLED=0
ENV GOOS=linux
ENV GOARCH=amd64
ENV GO111MODULE=on
ENV GOPROXY="https://goproxy.cn"

# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN go mod download && \
    go build -a -o admission-registry main.go && \
    upx admission-registry

FROM alpine:3.9.2
COPY --from=builder /workspace/admission-registry .
ENTRYPOINT ["/admission-registry"]

我们把认证作为pod运行在集群中

apiVersion: apps/v1
kind: Deployment
metadata:
  name: admission-deploy
  labels: 
    app: admission-deploy
spec:
  replicas: 1
  selector: 
    matchLabels: 
      app: admission-deploy
  template:
    metadata:
      labels:
        app: admission-deploy 
    spec:
      containers: 
        - name: whitelist
          image: webhook-validate-mutate:v1.0
          imagePullPolicy: IfNotPresent
          env: 
          - name: WHITELIST_REGISTRIES
            value: "docker.io,gcr.io"
          ports:
          - containerPort: 443
          volumeMounts: 
          - name: webhook-certs
            mountPath: /etc/webhook/certs
            readOnly: true
      volumes:
        - name: webhook-certs
          secret:
            secretName: admission-registry-tls
---
apiVersion: v1
kind: Service
metadata:
  name: admission-registry
  labels:
    app: admission-registry
spec:
  ports:
  - port: 443
    targetPort: 443
  selector:
    app: admission-deploy

在这一步之前需要创建secret，接着需要创建validatingwebhookconfigurations，mutatingwebhookconfigurations

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: admission-registry
webhooks:
- name: io.ydzs.admission-registry
  rules:
  - apiGroups:   [""]
    apiVersions: ["v1"]
    operations:  ["CREATE"]
    resources:   ["pods"]
  clientConfig:
    service:
      namespace: default
      name: admission-registry
      path: "/validate"
    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR2akNDQXFhZ0F3SUJBZ0lVUkZkVTlRNTZXenA0UFNwUHQ0U05sdm9LOVBZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFVcHBibWN4RURBT0JnTlZCQWNUQjBKbAphVXBwYm1jeEREQUtCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByCmRXSmxjbTVsZEdWek1CNFhEVEl6TURZd016QXlNVEV3TUZvWERUSTRNRFl3TVRBeU1URXdNRm93WlRFTE1Ba0cKQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFVcHBibWN4RURBT0JnTlZCQWNUQjBKbGFVcHBibWN4RERBSwpCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByZFdKbGNtNWxkR1Z6Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMXNKdWwzR3doYXM1NURiVU45VzMKMFpKSEN0TCszZ2tYdzFGT1ZRZ1ZvSkVQcHFhV3pMN2hTTElHenNpQkRyQm5sWVIxVWR0TzNieU52dmVjcnZ5Mgo1ZVh1MmlVcFJkc2Q2NVpGbmZQUVZYNkFWditUSVNVMGg1a0RrU0VmdFpJNm5jNDNkaU9xMVhpVE5vWS82RG9hClZVV0tHemVCcHp0NG1zY2RLYXdvUENvMFZibGdmUytvT1FEQTVPdGl3NTVPclNVYUlrb3NHazlnVkhsU1V1SVMKUUNhaG4zTkxHMG9oblhLWkVWSlpwQ25wZDgwejJPWFRXb3FwSEZpdXVoTjZtd3Z6RUJYaXY3ay96dG9zKzFwVApkYXd4WWMyUVFBVURTK0NubTdwWGE2Sk82eUhjV1d4ZWxwUENMRnBjQm0yN0tJM2tESW1OMVB6SEZaUHFDYlVHCitRSURBUUFCbzJZd1pEQU9CZ05WSFE4QkFmOEVCQU1DQVFZd0VnWURWUjBUQVFIL0JBZ3dCZ0VCL3dJQkFqQWQKQmdOVkhRNEVGZ1FVMEQzZ2RPUERwVzc5Q0ZFZVA1M1ZTK1JnNEVFd0h3WURWUjBqQkJnd0ZvQVUwRDNnZE9QRApwVzc5Q0ZFZVA1M1ZTK1JnNEVFd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLUUlmQUVVdUEzQmgzOTJ6VkNMCmxHY0NpMzJpeGRBdmpVeXRNaU5pNHpFRHZDNVJoU05CT3lKWmVqemswRmRzZGNXcU9LTEtJT1J2NEpXV1NIQ1EKaDUwM28rUUNBMGQ2RkJmSDNHMkV3ak5iWUJYOWphYkM5K05rSVpDT1U3b3RqSTd4NUxJSGVWanpGZWJXR0FnOQpYejJGQVd2eXlTK3FPVzRzc3N1djAyQW1KMjJqTWlXSHd2elBWTlFDdHZIVWpZNDFrTXlnUmEyYzlYRmN4Wms5CkY2em56ZWdpamlzUXJBVm9jR09KbXNwYVd6N3JlZmFKalYzVU1ZcnZLWGlDRVhHS1JBZ05KRUthTzN5dlBZN3cKaVVHODFTbGdwY1hJNEFjci9QN0FUMGVzc1AwU1NPU1E1aU44UVhqcWhNN0FhRHlKRmNNYU53QzJUcWk1bEpETgprZUE9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K

  admissionReviewVersions: ["v1"]
  sideEffects: None
---
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: admission-registry-mute
webhooks:
  - name: io.ydzs.admission-registry-mutate
    clientConfig:
      service:
        namespace: default
        name: admission-registry
        path: "/mutate"
      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR2akNDQXFhZ0F3SUJBZ0lVUkZkVTlRNTZXenA0UFNwUHQ0U05sdm9LOVBZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pURUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFVcHBibWN4RURBT0JnTlZCQWNUQjBKbAphVXBwYm1jeEREQUtCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByCmRXSmxjbTVsZEdWek1CNFhEVEl6TURZd016QXlNVEV3TUZvWERUSTRNRFl3TVRBeU1URXdNRm93WlRFTE1Ba0cKQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFVcHBibWN4RURBT0JnTlZCQWNUQjBKbGFVcHBibWN4RERBSwpCZ05WQkFvVEEyczRjekVQTUEwR0ExVUVDeE1HVTNsemRHVnRNUk13RVFZRFZRUURFd3ByZFdKbGNtNWxkR1Z6Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMXNKdWwzR3doYXM1NURiVU45VzMKMFpKSEN0TCszZ2tYdzFGT1ZRZ1ZvSkVQcHFhV3pMN2hTTElHenNpQkRyQm5sWVIxVWR0TzNieU52dmVjcnZ5Mgo1ZVh1MmlVcFJkc2Q2NVpGbmZQUVZYNkFWditUSVNVMGg1a0RrU0VmdFpJNm5jNDNkaU9xMVhpVE5vWS82RG9hClZVV0tHemVCcHp0NG1zY2RLYXdvUENvMFZibGdmUytvT1FEQTVPdGl3NTVPclNVYUlrb3NHazlnVkhsU1V1SVMKUUNhaG4zTkxHMG9oblhLWkVWSlpwQ25wZDgwejJPWFRXb3FwSEZpdXVoTjZtd3Z6RUJYaXY3ay96dG9zKzFwVApkYXd4WWMyUVFBVURTK0NubTdwWGE2Sk82eUhjV1d4ZWxwUENMRnBjQm0yN0tJM2tESW1OMVB6SEZaUHFDYlVHCitRSURBUUFCbzJZd1pEQU9CZ05WSFE4QkFmOEVCQU1DQVFZd0VnWURWUjBUQVFIL0JBZ3dCZ0VCL3dJQkFqQWQKQmdOVkhRNEVGZ1FVMEQzZ2RPUERwVzc5Q0ZFZVA1M1ZTK1JnNEVFd0h3WURWUjBqQkJnd0ZvQVUwRDNnZE9QRApwVzc5Q0ZFZVA1M1ZTK1JnNEVFd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLUUlmQUVVdUEzQmgzOTJ6VkNMCmxHY0NpMzJpeGRBdmpVeXRNaU5pNHpFRHZDNVJoU05CT3lKWmVqemswRmRzZGNXcU9LTEtJT1J2NEpXV1NIQ1EKaDUwM28rUUNBMGQ2RkJmSDNHMkV3ak5iWUJYOWphYkM5K05rSVpDT1U3b3RqSTd4NUxJSGVWanpGZWJXR0FnOQpYejJGQVd2eXlTK3FPVzRzc3N1djAyQW1KMjJqTWlXSHd2elBWTlFDdHZIVWpZNDFrTXlnUmEyYzlYRmN4Wms5CkY2em56ZWdpamlzUXJBVm9jR09KbXNwYVd6N3JlZmFKalYzVU1ZcnZLWGlDRVhHS1JBZ05KRUthTzN5dlBZN3cKaVVHODFTbGdwY1hJNEFjci9QN0FUMGVzc1AwU1NPU1E1aU44UVhqcWhNN0FhRHlKRmNNYU53QzJUcWk1bEpETgprZUE9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K

    rules: 
      - operations: ["CREATE"]
        apiGroups: ["apps",""]
        apiVersions: ["v1"]
        resources: ["deployments", "services"]
    admissionReviewVersions: ["v1"]
    sideEffects: None