apiextensions CRD实现分析

k8s里有很多提前定义好的资源，比如pod资源，使用PodSpec表示，每创建一个pod就相当于对pod资源的一个实例化。

//pkg/apis/core/types.go
// PodSpec is a description of a pod
type PodSpec struct {
	Volumes []Volume
	InitContainers []Container
	Containers []Container
	EphemeralContainers []EphemeralContainer
	RestartPolicy RestartPolicy
	NodeSelector map[string]string
	NodeName string
	...
}

那如果想创建一种新的资源怎么实现呢，当然可以修改源码重新编译来添加，但是这样是不优雅的，而且很繁琐的。这时就可以使用crd来实现了，crd的目的就是创建新的资源类型，通过CustomResourceDefinitionSpec描述新资源信息。新资源类型相当于是crd的实例，后面再根据新资源类型创建它的实例。

//k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1/types.go
type CustomResourceDefinitionSpec struct {
	Group string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Names CustomResourceDefinitionNames `json:"names" protobuf:"bytes,3,opt,name=names"`
	Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"`
	Versions []CustomResourceDefinitionVersion `json:"versions" protobuf:"bytes,7,rep,name=versions"`
	...
}

安装crd路由
前面已经说过安装crd路由大致流程，这里再复述一下，

//k8s.io/apiextensions/pkg/apiserver/apiserver.go
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
	//k8s.io/apiserver/pkg/server/config.go
	genericServer, err := c.GenericConfig.New("apiextensions-apiserver", delegationTarget)
	...
	s := &CustomResourceDefinitions{
		GenericAPIServer: genericServer,
	}

	apiResourceConfig := c.GenericConfig.MergedResourceConfig
	//const GroupName = "apiextensions.k8s.io"
	//Scheme为apiExtensionsServer的资源注册表，位于k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go
	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs)
	if apiResourceConfig.VersionEnabled(v1.SchemeGroupVersion) {
		storage := map[string]rest.Storage{}
		//crd资源背后的storage
		customResourceDefinitionStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter)

		storage["customresourcedefinitions"] = customResourceDefinitionStorage
		storage["customresourcedefinitions/status"] = customresourcedefinition.NewStatusREST(Scheme, customResourceDefinitionStorage)

		apiGroupInfo.VersionedResourcesStorageMap[v1.SchemeGroupVersion.Version] = storage
	}
	//安装/apis/apiextensions.k8s.io/customresourcedefinitions路由到genericServer.handler.GoRestfulContainer，
	//通过此路由请求创建crd，对应的handler为store.create/get等，直接和etcd交互
	s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo)

	crdClient, err := clientset.NewForConfig(s.GenericAPIServer.LoopbackClientConfig)
	s.Informers = externalinformers.NewSharedInformerFactory(crdClient, 5*time.Minute)
	...
	crdHandler, err := NewCustomResourceDefinitionHandler(
		versionDiscoveryHandler,
		groupDiscoveryHandler,
		s.Informers.Apiextensions().V1().CustomResourceDefinitions(),
		...
	)
	//安装/apis/路由到genericServer.handler.NonGoRestfulMux，
	//用于通过前缀apis拦截对自定义资源的增删改查，对应的handler为crdHandler
	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler)
	s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)

自定义资源实例的增删改查
向/apis/apiextensions.k8s.io/v1/customresourcedefinitions发起请求可以创建自定义资源，将相关信息写入etcd中。
向/apis/自定义group/版本/namespace/自定义资源发起请求时，最终会匹配到apiextensions安装到NonGoRestfulMux上的/apis/a路由，对应的处理函数为crdHandler.ServeHTTP，如下所示

//对自定义资源的增删改查操作处理的handler
func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
	ctx := req.Context()
	requestInfo, ok := apirequest.RequestInfoFrom(ctx)
	//如果不是对资源的请求
	if !requestInfo.IsResourceRequest {
		pathParts := splitPath(requestInfo.Path)
		// only match /apis/<group>/<version>
		// only registered under /apis
		//如果是对/apis/<group>/<version>的请求，则返回version下的所有自定义资源信息
		if len(pathParts) == 3 {
			if !r.hasSynced() {
				responsewriters.ErrorNegotiated(serverStartingError(), Codecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req)
				return
			}
			r.versionDiscoveryHandler.ServeHTTP(w, req)
			return
		}
		// only match /apis/<group>
		//如果是对/apis/<group>的请求，则返回group下的所有版本信息
		if len(pathParts) == 2 {
			if !r.hasSynced() {
				responsewriters.ErrorNegotiated(serverStartingError(), Codecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req)
				return
			}
			r.groupDiscoveryHandler.ServeHTTP(w, req)
			return
		}

		r.delegate.ServeHTTP(w, req)
		return
	}
	
	crdName := requestInfo.Resource + "." + requestInfo.APIGroup
	//根据crd名字获取自定义资源
	crd, err := r.crdLister.Get(crdName)

	//根据crd id创建或者查找crd对应的crdinfo
	crdInfo, err := r.getOrCreateServingInfoFor(crd.UID, crd.Name)
		storageMap := r.customStorage.Load().(crdStorageMap)
		if ret, ok := storageMap[uid]; ok {
			return ret, nil
		}

		crd, err := r.crdLister.Get(name)
		storageMap = r.customStorage.Load().(crdStorageMap)
		if ret, ok := storageMap[crd.UID]; ok {
			return ret, nil
		}

		for _, v := range crd.Spec.Versions {
			//创建自定义资源的storage
			storages[v.Name] = customresource.NewStorage(
				resource.GroupResource(),
				kind,
				...
				)
		}

		ret := &crdInfo{
			spec:                &crd.Spec,
			acceptedNames:       &crd.Status.AcceptedNames,
			storages:            storages,
			requestScopes:       requestScopes,
			scaleRequestScopes:  scaleScopes,
			statusRequestScopes: statusScopes,
			deprecated:          deprecated,
			warnings:            warnings,
			storageVersion:      storageVersion,
			waitGroup:           &utilwaitgroup.SafeWaitGroup{},
		}

		// Copy because we cannot write to storageMap without a race
		// as it is used without locking elsewhere.
		storageMap2 := storageMap.clone()

		storageMap2[crd.UID] = ret
		r.customStorage.Store(storageMap2)

		return ret, nil

	switch {
	case subresource == "status" && subresources != nil && subresources.Status != nil:
		handlerFunc = r.serveStatus(w, req, requestInfo, crdInfo, terminating, supportedTypes)
	case subresource == "scale" && subresources != nil && subresources.Scale != nil:
		handlerFunc = r.serveScale(w, req, requestInfo, crdInfo, terminating, supportedTypes)
	case len(subresource) == 0:
		handlerFunc = r.serveResource(w, req, requestInfo, crdInfo, crd, terminating, supportedTypes)
			requestScope := crdInfo.requestScopes[requestInfo.APIVersion]
			storage := crdInfo.storages[requestInfo.APIVersion].CustomResource
			//根据verb返回不同的处理函数
			switch requestInfo.Verb {
			case "get":
				return handlers.GetResource(storage, requestScope)
			case "list":
				forceWatch := false
				return handlers.ListResource(storage, storage, requestScope, forceWatch, r.minRequestTimeout)
			case "watch":
				forceWatch := true
				return handlers.ListResource(storage, storage, requestScope, forceWatch, r.minRequestTimeout)
			case "create":
				return handlers.CreateResource(storage, requestScope, r.admission)
					createHandler(&namedCreaterAdapter{r}, scope, admission, false)
			...
			}
	}

	if handlerFunc != nil {
		handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, deprecated, "", handlerFunc)
		handler := genericfilters.WithWaitGroup(handlerFunc, longRunningFilter, crdInfo.waitGroup)
			return withWaitGroup(handler, longRunning, wg, isRequestExemptFromRetryAfter)
				return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
					handler.ServeHTTP(w, req)
				}
		//调用 handlerFunc.ServeHTTP，HandlerFunc 本身也实现了ServeHTTP函数
		handler.ServeHTTP(w, req)
		return
	}

// ServeHTTP calls f(w, r).
func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) {
	f(w, r)
}

如果verb是get，则最后会和etcd交互获取相应的信息

//k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go
func (r *REST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	//k8s.io/apiserver/pkg/registry/generic/registry/store.go
	//和etcd交互，获取信息
	o, err := r.Store.Get(ctx, name, options)
	if err != nil {
		return nil, err
	}
	if u, ok := o.(*unstructured.Unstructured); ok {
		shallowCopyObjectMeta(u)
	}
	return o, nil
}