自动生成webhook组件证书

bebc

已于 2023-01-18 20:17:54 修改

阅读量1.5k

点赞数

分类专栏： k8s go client-go 文章标签： golang k8s 组件

于 2023-01-18 20:17:24 首次发布

本文链接：https://blog.csdn.net/u013694670/article/details/128649796

版权

k8s 同时被 3 个专栏收录

11 篇文章 2 订阅

订阅专栏

7 篇文章 0 订阅

订阅专栏

client-go

5 篇文章 1 订阅

订阅专栏

一、碰到的问题

在开发监控operator的时候，由于项目中有webhook功能，因此在部署的时候需要生成访问webhook所需的证书，一般证书生成有以下两种方式：
1.shell脚本实现
2.cert-manager管理

由于项目需要通过应用市场直接部署至集群中，因此以上两种方式在这个方案中都不是最优的解。

shell脚本无法集成到应用市场中，cert-manager不能保证用户集群一定会安装，如果集成到helm中会显的很笨重。在参考了一些开源项目后，决定webhook证书最好能集成到组件中自动生成，并通过项目启动参数开关来交给用户控制。

二、功能实现

生成证书：

//创建根证书
func (w *WebhookTls) createCACert() (*KeyPairArtifacts, error) {
	begin := time.Now().Add(-1 * time.Hour)
	end := begin.Add(certValidityDuration)

	templ := &x509.Certificate{
		SerialNumber: big.NewInt(0),
		Subject: pkix.Name{
			CommonName:   CAName,
			Organization: []string{CAOrganization},
		},
		DNSNames: []string{
			CAName,
		},
		NotBefore:             begin,
		NotAfter:              end,
		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}
	//生成根证书私钥
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, errors.Wrap(err, "generating key")
	}
	//生成根证书
	der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
	if err != nil {
		return nil, errors.Wrap(err, "creating certificate")
	}
	//certPEM, keyPEM, err := pemEncode(der, key)
	//if err != nil {
	//	return nil, errors.Wrap(err, "encoding PEM")
	//}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, errors.Wrap(err, "parsing certificate")
	}

	return &KeyPairArtifacts{Cert: cert, Key: key}, nil
}

//创建服务器证书
func (w *WebhookTls) createCertPEM() (*KeyPairArtifacts, error) {
	ca, err := w.createCACert()
	if err != nil {
		return nil, err
	}
	begin := time.Now().Add(-1 * time.Hour)
	end := begin.Add(certValidityDuration)
	DNSName := fmt.Sprintf("%s.%s.svc", serviceName, w.Namespace)

	templ := &x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			CommonName: DNSName,
		},
		DNSNames: []string{
			DNSName,
		},
		NotBefore:             begin,
		NotAfter:              end,
		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	//创建webhook服务器私钥
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, errors.Wrap(err, "generating key")
	}
	//创建webhook服务器证书
	der, err := x509.CreateCertificate(rand.Reader, templ, ca.Cert, key.Public(), ca.Key)
	if err != nil {
		return nil, errors.Wrap(err, "creating certificate")
	}
	certPEM, keyPEM, err := pemEncode(der, key)
	if err != nil {
		return nil, errors.Wrap(err, "encoding PEM")
	}
	return &KeyPairArtifacts{CertPEM: certPEM, KeyPEM: keyPEM}, nil
}

func pemEncode(certificateDER []byte, key *rsa.PrivateKey) ([]byte, []byte, error) {
	certBuf := &bytes.Buffer{}
	if err := pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}); err != nil {
		return nil, nil, errors.Wrap(err, "encoding cert")
	}
	keyBuf := &bytes.Buffer{}
	if err := pem.Encode(keyBuf, &pem.Block{Type: "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil {
		return nil, nil, errors.Wrap(err, "encoding key")
	}
	return certBuf.Bytes(), keyBuf.Bytes(), nil
}

创建服务器所需证书：

//创建webhook服务tls证书
func (w *WebhookTls) createTls(cert []byte, key []byte) error {
	certDir := w.CertDir
	if _, err := os.Stat(certDir); os.IsNotExist(err) {
		if err := os.MkdirAll(certDir, 0700); err != nil {
			return err
		}
	}

	if err := ioutil.WriteFile(path.Join(certDir, certName), cert, 0600); err != nil {
		return err
	}
	if err := ioutil.WriteFile(path.Join(certDir, keyName), key, 0600); err != nil {
		return err
	}

	return nil
}

更新caBundle字段：

//更新webhookconfig caBundle字段
func (w *WebhookTls) updateCaBundle(cert []byte) error {
	MutatingWebhook, err := w.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().
		Get(context.Background(), webhookName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	for i := range MutatingWebhook.Webhooks {
		MutatingWebhook.Webhooks[i].ClientConfig.CABundle = cert
	}

	_, err = w.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().
		Update(context.Background(), MutatingWebhook, metav1.UpdateOptions{})

	if err != nil {
		return err
	}

	return nil
}

启动webhook服务器：

//ca
webhookTls := pkg.NewWebHookTls(namespace, clientSet, certDir)
err = webhookTls.RunWebHookTls()
if err != nil {
	setupLog.Error(err, "unable to init webhookTls")
	os.Exit(1)
}

//server
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
	Scheme:                 scheme,
	MetricsBindAddress:     metricsAddr,
	Port:                   9443,
	LeaderElection:         enableLeaderElection,
	HealthProbeBindAddress: probeAddr,
	LeaderElectionID:       "3f08ca71.tmc-gitlab.bebc.com",
})
if err != nil {
	setupLog.Error(err, "unable to start manager")
	os.Exit(1)
}

if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
	setupLog.Error(err, "unable to set up health check")
	os.Exit(1)
}
if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
	setupLog.Error(err, "unable to set up ready check")
	os.Exit(1)
}

mgr.GetWebhookServer().CertDir = certDir
mgr.GetWebhookServer().Register("/mutate", &webhook.Admission{Handler: &pkg.PodLabels{Client: mgr.GetClient(),
	Log: setupLog.WithName("webhook")}})

setupLog.Info("starting manager")

if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil {
	setupLog.Error(err, "problem running manager")
	os.Exit(1)
}

三、验证

kubectl apply -f example.yaml

查看服务日志：

[root@kind-master ~]# kubectl -n monitoring logs monitoring-operator-857dbdd495-8ks92 
2023-01-14T05:09:45Z	INFO	controller-runtime.metrics	Metrics server is starting to listen	{"addr": ":8080"}
2023-01-14T05:09:45Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/mutate"}
2023-01-14T05:09:45Z	INFO	monitoring-controller	starting manager
2023-01-14T05:09:45Z	INFO	controller-runtime.webhook.webhooks	Starting webhook server
2023-01-14T05:09:45Z	INFO	Starting server	{"path": "/metrics", "kind": "metrics", "addr": "[::]:8080"}
2023-01-14T05:09:45Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2023-01-14T05:09:45Z	INFO	Starting server	{"kind": "health probe", "addr": "[::]:8081"}
2023-01-14T05:09:45Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2023-01-14T05:09:45Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2023-01-14T05:10:05Z	DEBUG	controller-runtime.webhook.webhooks	received request	{"webhook": "/mutate", "UID": "f769223f-c891-4cb8-83de-cffc27531732", "kind": "/v1, Kind=Pod", "resource": {"group":"","version":"v1","resource":"pods"}}
2023-01-14T05:10:05Z	INFO	monitoring-controller.webhook	inject annotations	{"pod name": ""}
2023-01-14T05:10:05Z	DEBUG	controller-runtime.webhook.webhooks	wrote response	{"webhook": "/mutate", "code": 200, "reason": "", "UID": "f769223f-c891-4cb8-83de-cffc27531732", "allowed": true}

查看pod annotation:

[root@kind-master ~]# kubectl get pods webhook-example-d6898c896-8ng46 -o yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    cni.projectcalico.org/containerID: 4cb2a9f5810471cf4873a0f5d9c863f9196f04763227c9af308a6a89b3670a70
    cni.projectcalico.org/podIP: 172.21.68.159/32
    cni.projectcalico.org/podIPs: 172.21.68.159/32
    //注入的annotation
    prometheus.io/port: "8080"
  creationTimestamp: "2023-01-14T05:10:06Z"
  generateName: webhook-example-d6898c896-
  labels:

验证成功。

四、总结

通过在组件中自动生成自签名证书，解决webhook部署时所需的证书问题。之后在参考开源项目时，发现有自签名证书生成镜像k8s.gcr.io/ingress-nginx/kube-webhook-certgen，现成的轮子，可以集成到helm的hook中，这样项目中的代码只需要根据配置中的secret生成服务器证书就行。

完整项目路径：https://github.com/bebc/webhook-tls