kubelet grpc实现

kubelet grpc

kubelet中用到grpc主要是操作runtime，kubelet截止到目前的版本1.16.0支持多种runtime：Docker，CRI-O，Containerd，Other CRI runtimes: frakti。这里主要讲的是docker runtime的grcp实现，实现的大致流程如下：kubelet在启动的时候，封装了关于runtime的操作的grpc server（通过docker的client）和client端，然后启动server端，完成server端的启动之后，在初始化对应server端的client，封装在kubelet的runtimeService和imageService中，后续对pod相关的容器的操作，都是通过runtimeService和imageService去操作的完成的，至于为什么kubelet不直接调用docker的client去实现container相关的功能，而是封装一层grpc server和client，其主要目的是为了适配后端不同的runtime。

grpc server 接口定义

type RuntimeServiceServer interface {
	Version(context.Context, *VersionRequest) (*VersionResponse, error)
	RunPodSandbox(context.Context, *RunPodSandboxRequest) (*RunPodSandboxResponse, error)
	StopPodSandbox(context.Context, *StopPodSandboxRequest) (*StopPodSandboxResponse, error)
	RemovePodSandbox(context.Context, *RemovePodSandboxRequest) (*RemovePodSandboxResponse, error)
	PodSandboxStatus(context.Context, *PodSandboxStatusRequest) (*PodSandboxStatusResponse, error)
	ListPodSandbox(context.Context, *ListPodSandboxRequest) (*ListPodSandboxResponse, error)
	CreateContainer(context.Context, *CreateContainerRequest) (*CreateContainerResponse, error)
	StartContainer(context.Context, *StartContainerRequest) (*StartContainerResponse, error)
	StopContainer(context.Context, *StopContainerRequest) (*StopContainerResponse, error)
	RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error)
	ListContainers(context.Context, *ListContainersRequest) (*ListContainersResponse, error)
	ContainerStatus(context.Context, *ContainerStatusRequest) (*ContainerStatusResponse, error)
	UpdateContainerResources(context.Context, *UpdateContainerResourcesRequest) (*UpdateContainerResourcesResponse, error)
	ReopenContainerLog(context.Context, *ReopenContainerLogRequest) (*ReopenContainerLogResponse, error)
	ExecSync(context.Context, *ExecSyncRequest) (*ExecSyncResponse, error)
	Exec(context.Context, *ExecRequest) (*ExecResponse, error)
	Attach(context.Context, *AttachRequest) (*AttachResponse, error)
	PortForward(context.Context, *PortForwardRequest) (*PortForwardResponse, error)
	ContainerStats(context.Context, *ContainerStatsRequest) (*ContainerStatsResponse, error)
	ListContainerStats(context.Context, *ListContainerStatsRequest) (*ListContainerStatsResponse, error)
	UpdateRuntimeConfig(context.Context, *UpdateRuntimeConfigRequest) (*UpdateRuntimeConfigResponse, error)
	Status(context.Context, *StatusRequest) (*StatusResponse, error)
}


type ImageServiceServer interface {
	ListImages(context.Context, *ListImagesRequest) (*ListImagesResponse, error)
	ImageStatus(context.Context, *ImageStatusRequest) (*ImageStatusResponse, error)
	PullImage(context.Context, *PullImageRequest) (*PullImageResponse, error)
	RemoveImage(context.Context, *RemoveImageRequest) (*RemoveImageResponse, error)
	ImageFsInfo(context.Context, *ImageFsInfoRequest) (*ImageFsInfoResponse, error)
}

相关的接口定义以及grpc的代码所在位置：k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.pb.go

grpc server 接口实现

grpc server的代码实现主要在以下几个文件中：
k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_image.go实现接口 ImageServiceServer

k8s.io/kubernetes/pkg/kubelet/dockershim/docker_sandbox.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_container.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_streaming.go
k8s.io/kubernetes/pkg/kubelet/dockershim/docker_logs.go
实现RuntimeServiceServer接口

grpc server/client 实例化以及使用

func NewMainKubelet(...) (*Kubelet, error){
	...
	switch containerRuntime {
	case kubetypes.DockerContainerRuntime:
		// Create and start the CRI shim running as a grpc server.
		streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
		ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
			&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
		if err != nil {
			return nil, err
		}
		if crOptions.RedirectContainerStreaming {
			klet.criHandler = ds
		}

		// The unix socket for kubelet <-> dockershim communication.
		klog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
			remoteRuntimeEndpoint,
			remoteImageEndpoint)
		klog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
		server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
		// Start 方法会吧ds的所有方法注册到grpc的server端
		if err := server.Start(); err != nil {
			return nil, err
		}

		// Create dockerLegacyService when the logging driver is not supported.
		supported, err := ds.IsCRISupportedLogDriver()
		if err != nil {
			return nil, err
		}
		if !supported {
			klet.dockerLegacyService = ds
			legacyLogProvider = ds
		}
	case kubetypes.RemoteContainerRuntime:
		// No-op.
		break
	default:
		return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
	}
	// 这里是grpc的client端的实例的实现
	runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout)
	if err != nil {
		return nil, err
	}
	klet.runtimeService = runtimeService
	...
	runtime, err := kuberuntime.NewKubeGenericRuntimeManager(
		kubecontainer.FilterEventRecorder(kubeDeps.Recorder),
		klet.livenessManager,
		seccompProfileRoot,
		containerRefManager,
		machineInfo,
		klet,
		kubeDeps.OSInterface,
		klet,
		httpClient,
		imageBackOff,
		kubeCfg.SerializeImagePulls,
		float32(kubeCfg.RegistryPullQPS),
		int(kubeCfg.RegistryBurst),
		kubeCfg.CPUCFSQuota,
		kubeCfg.CPUCFSQuotaPeriod,
		runtimeService,
		imageService,
		kubeDeps.ContainerManager.InternalContainerLifecycle(),
		legacyLogProvider,
		klet.runtimeClassManager,
	)
	...
}

在实例化kubelet类型struct的时候，会根据runtime的类型去创建对应的dockerService，然后创建的对应的关于runtime所有接口实现的grpc server端，并监听grpc链接请求，提供runtime操作的服务，到这里grpc server端的实例化已经完成。

紧接着是初始化kubelet的runtimeService和imageService，实际就是创建对应的grpc的client端。这个client端在后续的多个manager中都会用到。比如：syncPod后，需要创建对应的容器就是最终就是调用runtimeService和imageService去创建容器。