APIServerHandler及ServeHTTP流程分析

kube-apiserver创建了三种server，他们对应的都为同一个结构体GenericAPIServer，其中和处理http请求关系最大的是APIServerHandler，所以本文重点看一下APIServerHandler

//k8s.io/apiserver/pkg/server/genericapiserver.go
// GenericAPIServer contains state for a Kubernetes cluster api server.
type GenericAPIServer struct {
	...
	// "Outputs"
	// Handler holds the handlers being used by this API server
	Handler *APIServerHandler
	...
}

//k8s.io/apiserver/pkg/server/handler.go
//APIServerHandler中保存了多种handler，收到http请求时，按照如下顺序调用
// FullHandlerChain -> Director -> {GoRestfulContainer,NonGoRestfulMux} based on inspection of registered web services
type APIServerHandler struct {
	// FullHandlerChain is the one that is eventually served with.  It should include the full filter
	// chain and then call the Director.
	//实现handlerchain，将多个handler连接起来，串行执行
	FullHandlerChain http.Handler
	// The registered APIs.  InstallAPIs uses this.  Other servers probably shouldn't access this directly.
	//存放restful api路由等信息
	GoRestfulContainer *restful.Container
	// NonGoRestfulMux is the final HTTP handler in the chain.
	// It comes after all filters and the API handling
	// This is where other servers can attach handler to various parts of the chain.
	//根据http请求path做精确和前缀匹配找到对应的handler
	NonGoRestfulMux *mux.PathRecorderMux
    //最终处理http请求的handler，可以直接调用，也可以放在FullHandlerChain的最后一环
	Director http.Handler
}

type director struct {
	name               string
	goRestfulContainer *restful.Container
	nonGoRestfulMux    *mux.PathRecorderMux
}

APIServerHandler中的GoRestfulContainer，NonGoRestfulMux和director中的goRestfulContainer，nonGoRestfulMux是相同的，前者是注册api时引用，后者是处理http请求时引用。

创建GenericAPIServer

GenericAPIServer的创建由New函数完成，第一个参数指定server名字，第二个参数指定代理对象，其类型为DelegationTarget接口

type DelegationTarget interface {
	// UnprotectedHandler returns a handler that is NOT protected by a normal chain
	UnprotectedHandler() http.Handler
	...
	// ListedPaths returns the paths for supporting an index
	ListedPaths() []string
	// NextDelegate returns the next delegationTarget in the chain of delegations
	NextDelegate() DelegationTarget
	// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
	PrepareRun() preparedGenericAPIServer
}

GenericAPIServer实现了DelegationTarget接口，比如UnprotectedHandler

func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
	// when we delegate, we need the server we're delegating to choose whether or not to use gorestful
	return s.Handler.Director
}

New函数实现如下

//k8s.io/apiserver/pkg/server/config.go
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
	...
	//用于创建handlerchain，后面会单独分析
	handlerChainBuilder := func(handler http.Handler) http.Handler {
		//BuildHandlerChainFunc 为 DefaultBuildHandlerChain，用于创建handlerchain
		return c.BuildHandlerChainFunc(handler, c.Config)
	}
	//delegationTarget.UnprotectedHandler()获取代理对象的handler
	apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
	
	s := &GenericAPIServer{
		...
		Handler: apiServerHandler,

		listedPathProvider: apiServerHandler,
		...
		Version: c.Version,
	}
	...
	return s, nil

NewAPIServerHandler

//k8s.io/apiserver/pkg/server/handler.go
func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
	//生成非restful的多路复用器，用于根据http请求path的精确匹配和前缀匹配找到对应的handler
	nonGoRestfulMux := mux.NewPathRecorderMux(name)
	//将notFoundHandler保存到nonGoRestfulMux中，当前server处理不了http请求时，就会调用代理对象的Handler处理，实现链式调用
	if notFoundHandler != nil {
		nonGoRestfulMux.NotFoundHandler(notFoundHandler)
	}

	//创建Container，用于存放资源的restful api
	gorestfulContainer := restful.NewContainer()
	...
	//生成director，gorestfulContainer和nonGoRestfulMux保存到director中，同时也保存到APIServerHandler中
	director := director{
		name:               name,
		goRestfulContainer: gorestfulContainer,
		nonGoRestfulMux:    nonGoRestfulMux,
	}
	//生成APIServerHandler
	return &APIServerHandler{
		//handlerChainBuilder用于生成handlerchain，并将director放到handlerchain的最后，
		//意思为只有当http请求通过handlerchain时，才会调用director.ServeHTTP匹配路由
		FullHandlerChain:   handlerChainBuilder(director),
		GoRestfulContainer: gorestfulContainer,
		NonGoRestfulMux:    nonGoRestfulMux,
		Director:           director,
	}
}

DefaultBuildHandlerChain
DefaultBuildHandlerChain中创建了多种handlerchain，并将它们链接起来，当收到http请求时，依次经过这条链上handler的处理，
每个handler决定是否调用下一个handler，比如Authorization handler，只要授权失败就会终止执行后面的handler。
这里只列出来授权和认证两种handler。

//k8s.io/apiserver/pkg/server/config.go
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
	handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
	...
	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
	...
	return handler
}

还要注意的是，这些handler的注册顺序和执行顺序是相反的，比如上面的注册顺序为apiHandler -> Authorization handler -> Authentication handler，则执行顺序为Authentication handler -> Authorization handler -> apiHandler 。

下面以授权handler为例，看一下如何注册handler及何时执行handler。
WithAuthorization传入一个handler，返回另一个新的handler，当执行此新handler进行授权操作时，如果授权成功了则执行传进来的handler，否则终止执行并返回错误

//k8s.io/apiserver/pkg/endpoints/filters/authorization.go
// WithAuthorizationCheck passes all authorized requests on to handler, and returns a forbidden error otherwise.
func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {
	if a == nil {
		klog.Warning("Authorization is disabled")
		return handler
	}
	//返回handler
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		ae := request.AuditEventFrom(ctx)

		attributes, err := GetAuthorizerAttributes(ctx)
		if err != nil {
			responsewriters.InternalError(w, req, err)
			return
		}
		authorized, reason, err := a.Authorize(ctx, attributes)
		// an authorizer like RBAC could encounter evaluation errors and still allow the request, so authorizer decision is checked before error here.
		//授权结果为允许，则执行handler.ServeHTTP
		if authorized == authorizer.DecisionAllow {
			audit.LogAnnotation(ae, decisionAnnotationKey, decisionAllow)
			audit.LogAnnotation(ae, reasonAnnotationKey, reason)
			handler.ServeHTTP(w, req)
			return
		}
		if err != nil {
			audit.LogAnnotation(ae, reasonAnnotationKey, reasonError)
			responsewriters.InternalError(w, req, err)
			return
		}

		klog.V(4).InfoS("Forbidden", "URI", req.RequestURI, "Reason", reason)
		audit.LogAnnotation(ae, decisionAnnotationKey, decisionForbid)
		audit.LogAnnotation(ae, reasonAnnotationKey, reason)
		responsewriters.Forbidden(ctx, attributes, w, req, reason, s)
	})
}

启动server

在kube-apiserver的主流程中，通过CreateServerChain创建三个服务链后，开始启动server监听http请求

//cmd/kube-apiserver/app/server.go
// Run runs the specified APIServer.  This should never exit.
func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error {
	// To help debugging, immediately log version
	klog.Infof("Version: %+v", version.Get())
	//创建服务链，并返回链上最后添加的server
	server, err := CreateServerChain(completeOptions, stopCh)
	...
	//启动最后添加的server
	return prepared.Run(stopCh)
}

prepared.Run运行的是如下函数

//k8s.io/apiserver/pkg/server/genericapiserver.go
// Run spawns the secure http server. It only returns if stopCh is closed
// or the secure port cannot be listened on initially.
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
	...
	stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(stopHttpServerCh, shutdownTimeout)
	...
}

最终调用标准库接口启动监听

//k8s.io/apiserver/pkg/server/genericapiserver.go
// NonBlockingRun spawns the secure http server. An error is
// returned if the secure port cannot be listened on.
// The returned channel is closed when the (asynchronous) termination is finished.
func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
	...
	if s.SecureServingInfo != nil && s.Handler != nil {
		var err error
		//ServeWithListenerStopped最终调用go标准库里的http接口启动监听
		stoppedCh, listenerStoppedCh, err = s.SecureServingInfo.ServeWithListenerStopped(s.Handler, shutdownTimeout, internalStopCh)
	}
	...
}

ServeHTTP

ServeHTTP是用来处理http请求的入口，前面提到的FullHandlerChain和Director handler必须实现如下接口才能处理http请求

//net/http.go
type Handler interface {
	ServeHTTP(ResponseWriter, *Request)
}

APIServerHandler也实现了Handler接口，并且是处理http请求的第一个环节

//k8s.io/apiserver/pkg/server/handler.go
// ServeHTTP makes it an http.Handler
func (a *APIServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//开始执行handlerchain上的handler，比如WithAuthorization返回的handler
	a.FullHandlerChain.ServeHTTP(w, r)
}

如果handlerchain上的handler执行全部成功，则最后调用到director handler的ServeHTTP

//k8s.io/apiserver/pkg/server/handler.go
func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path

	//获取container上注册的所有webservice
	// check to see if our webservices want to claim this path
	for _, ws := range d.goRestfulContainer.RegisteredWebServices() {
		switch {
		case ws.RootPath() == "/apis":
			// if we are exactly /apis or /apis/, then we need special handling in loop.
			// normally these are passed to the nonGoRestfulMux, but if discovery is enabled, it will go directly.
			// We can't rely on a prefix match since /apis matches everything (see the big comment on Director above)
			if path == "/apis" || path == "/apis/" {
				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
				// TODO fix gorestful, remove TPRs, or stop using gorestful
				d.goRestfulContainer.Dispatch(w, req)
				return
			}

		//如果请求路径包含ws的rootpath。ws的rootpath为请求路径去掉最后一个/及其后面的内容，
		//比如对于/api/v1/pod，则rootpath为/api/v1
		case strings.HasPrefix(path, ws.RootPath()):
			// ensure an exact match or a path boundary match
			//匹配到ws
			if len(path) == len(ws.RootPath()) || path[len(ws.RootPath())] == '/' {
				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
				// TODO fix gorestful, remove TPRs, or stop using gorestful
				//匹配到ws，将请求分发到具体的ws进行处理
				d.goRestfulContainer.Dispatch(w, req)
				return
			}
		}
	}

	//如果goRestfulContainer匹配不到，则下一步交给nonGoRestfulMux处理
	// if we didn't find a match, then we just skip gorestful altogether
	klog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path)
	d.nonGoRestfulMux.ServeHTTP(w, req)
}

nonGoRestfulMux.ServeHTTP的处理主要是根据path进行精确匹配和前缀匹配

//k8s.io/apiserver/pkg/server/mux/pathrecorder.go
// ServeHTTP makes it an http.Handler
func (m *PathRecorderMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.Load().(*pathHandler).ServeHTTP(w, r)
}

// ServeHTTP makes it an http.Handler
func (h *pathHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//精确匹配
	if exactHandler, ok := h.pathToHandler[r.URL.Path]; ok {
		klog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path)
		exactHandler.ServeHTTP(w, r)
		return
	}

	//前缀匹配
	for _, prefixHandler := range h.prefixHandlers {
		if strings.HasPrefix(r.URL.Path, prefixHandler.prefix) {
			klog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix)
			prefixHandler.handler.ServeHTTP(w, r)
			return
		}
	}

	//仍然没有匹配上，则调用notFoundHandler.ServeHTTP
	klog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path)
	h.notFoundHandler.ServeHTTP(w, r)
}

notFoundHandler为被代理对象handler，由delegationTarget.UnprotectedHandler()获取，如果代理对象为GenericAPIServer，则获取到Director

func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
	// when we delegate, we need the server we're delegating to choose whether or not to use gorestful
	return s.Handler.Director
}

所以如果当前处理的是aggregatorserver，则notFoundHandler.ServeHTTP执行的是kube-apiserver的Director.ServeHTTP，
如果kube-apiserver仍然处理不了，则notFoundHandler.ServeHTTP执行的是extensionserver的Director.ServeHTTP，最后仍然处理不了，则由NotFound处理，即返回404，这也就是apiserver的服务链

// NotFound replies to the request with an HTTP 404 not found error.
func NotFound(w ResponseWriter, r *Request) { Error(w, "404 page not found", StatusNotFound) }

// NotFoundHandler returns a simple request handler
// that replies to each request with a ``404 page not found'' reply.
func NotFoundHandler() Handler { return HandlerFunc(NotFound) }

总结一下，收到http请求后的处理流程为：

aggregatorserver FullHandlerChain -> aggregatorserver director -> apiserver director -> extensionserver director -> NotFound

这里要注意的一点是，这三个server都实现了FullHandlerChain，但是只有在最开始aggregatorserver处被调用了。