【kubernetes/k8s源码分析】kubernetes volume plugin源码分析

plugin调用函数：

      文章https://blog.csdn.net/zhonglinzhang/article/details/82800287 分析volume manager调用

• reconcile -> 
• MountVolume ->
• NewMouter根据插件调用 ->
• 执行plugin的SetUp

 

volumePlugin接口

• 存储提供的扩展接口, 包含了各类存储提供者的plugin实现
• 实现自定义的Plugins 可以通过FlexVolume(K8s 1.8版本)
• 容器存储接口 CSI（Container Storage Interface）
• 支持这套标准以后，K8S和存储提供者之间将彻底解耦，这一功能让新的卷插件的安装像部署Pod 一样简单，第三方存储开发的代码也不需要加入核心的Kubernetes代码中。

// VolumePlugin is an interface to volume plugins that can be used on a
// kubernetes node (e.g. by kubelet) to instantiate and manage volumes.
type VolumePlugin interface {
	
	Init(host VolumeHost) error

	GetPluginName() string

	GetVolumeName(spec *Spec) (string, error)

	CanSupport(spec *Spec) bool


	RequiresRemount() bool


	NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error)

	NewUnmounter(name string, podUID types.UID) (Unmounter, error)

	ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error)

	SupportsMountOption() bool

	SupportsBulkVolumeVerification() bool
}

 

1. kubelet 启动注册volume plugins

•      cmd/kubelet/app/server.go中run函数调用UnsecuredDependencies函数注册volume plugins，主要函数为ProbeVolumePlugins()

func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, error) {


	return &kubelet.Dependencies{
		
		Mounter:             mounter,
		NetworkPlugins:      ProbeNetworkPlugins(s.CNIConfDir, s.CNIBinDir),
		OOMAdjuster:         oom.NewOOMAdjuster(),
		OSInterface:         kubecontainer.RealOS{},
		Writer:              writer,
		VolumePlugins:       ProbeVolumePlugins(),

}

   1.1 ProbeVolumePlugins函数

•     注册到结构体为volume.volumePlugin列表，接口如VolumePlugin所示，主要讲解host_path，以及nfs

func ProbeVolumePlugins() []volume.VolumePlugin {
	allPlugins := []volume.VolumePlugin{}

	// The list of plugins to probe is decided by the kubelet binary, not
	// by dynamic linking or other "magic".  Plugins will be analyzed and
	// initialized later.
	//
	// Kubelet does not currently need to configure volume plugins.
	// If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig
	allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, empty_dir.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
	allPlugins = append(allPlugins, host_path.ProbeVolumePlugins(volume.VolumeConfig{})...)
	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
	
	if utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) {
		allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
	}
	return allPlugins
}

 

2. kubelet 初始化volume plugins

• pkg/kubelet/kubelet.go中NewMainKubelet函数中初始化volume plugin
• mountpod.NewMnager函数中kubelet的root  dir为 /var/lib/kubelet，目录下面有文件plugin-containers，plugins，pods
• InitPlugins初始化注册的所有volune plugin(2.1讲解)

func NewInitializedVolumePluginMgr(
	kubelet *Kubelet,
	secretManager secret.Manager,
	configMapManager configmap.Manager,
	plugins []volume.VolumePlugin,
	prober volume.DynamicPluginProber) (*volume.VolumePluginMgr, error) {

	mountPodManager, err := mountpod.NewManager(kubelet.getRootDir(), kubelet.podManager)
	if err != nil {
		return nil, err
	}
	kvh := &kubeletVolumeHost{
		kubelet:          kubelet,
		volumePluginMgr:  volume.VolumePluginMgr{},
		secretManager:    secretManager,
		configMapManager: configMapManager,
		mountPodManager:  mountPodManager,
	}

	if err := kvh.volumePluginMgr.InitPlugins(plugins, prober, kvh); err != nil {
		return nil, fmt.Errorf(
			"Could not initialize volume plugins for KubeletVolumePluginMgr: %v",
			err)
	}

	return &kvh.volumePluginMgr, nil
}

   2.1 InitPlugins函数

• 路径pkg/volume/plugins.go
• 调用各个plugin的init初始化

func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error {

	if pm.plugins == nil {
		pm.plugins = map[string]VolumePlugin{}
	}

	allErrs := []error{}
	for _, plugin := range plugins {
		name := plugin.GetPluginName()
		if errs := validation.IsQualifiedName(name); len(errs) != 0 {
			allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")))
			continue
		}

		if _, found := pm.plugins[name]; found {
			allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name))
			continue
		}
		err := plugin.Init(host)
		if err != nil {
			glog.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error())
			allErrs = append(allErrs, err)
			continue
		}
		pm.plugins[name] = plugin
		glog.V(1).Infof("Loaded volume plugin %q", name)
	}
	return utilerrors.NewAggregate(allErrs)
}

 

VolumeHost 接口

    路径pkg/volume/plugins.go

// VolumeHost is an interface that plugins can use to access the kubelet.
type VolumeHost interface {
	// GetPluginDir returns the absolute path to a directory under which
	// a given plugin may store data.  This directory might not actually
	// exist on disk yet.  For plugin data that is per-pod, see
	// GetPodPluginDir().
	GetPluginDir(pluginName string) string

	// GetVolumeDevicePluginDir returns the absolute path to a directory
	// under which a given plugin may store data.
	// ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/
	GetVolumeDevicePluginDir(pluginName string) string

	// GetPodsDir returns the absolute path to a directory where all the pods
	// information is stored
	GetPodsDir() string

	// GetPodVolumeDir returns the absolute path a directory which
	// represents the named volume under the named plugin for the given
	// pod.  If the specified pod does not exist, the result of this call
	// might not exist.
	GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string

	// GetPodPluginDir returns the absolute path to a directory under which
	// a given plugin may store data for a given pod.  If the specified pod
	// does not exist, the result of this call might not exist.  This
	// directory might not actually exist on disk yet.
	GetPodPluginDir(podUID types.UID, pluginName string) string

 

VolumeConfig结构体

type VolumeConfig struct {
	// RecyclerPodTemplate is pod template that understands how to scrub clean
	// a persistent volume after its release. The template is used by plugins
	// which override specific properties of the pod in accordance with that
	// plugin. See NewPersistentVolumeRecyclerPodTemplate for the properties
	// that are expected to be overridden.
	RecyclerPodTemplate *v1.Pod

	// RecyclerMinimumTimeout is the minimum amount of time in seconds for the
	// recycler pod's ActiveDeadlineSeconds attribute. Added to the minimum
	// timeout is the increment per Gi of capacity.
	RecyclerMinimumTimeout int

	// RecyclerTimeoutIncrement is the number of seconds added to the recycler
	// pod's ActiveDeadlineSeconds for each Gi of capacity in the persistent
	// volume. Example: 5Gi volume x 30s increment = 150s + 30s minimum = 180s
	// ActiveDeadlineSeconds for recycler pod
	RecyclerTimeoutIncrement int

	// PVName is name of the PersistentVolume instance that is being recycled.
	// It is used to generate unique recycler pod name.
	PVName string

	// OtherAttributes stores config as strings.  These strings are opaque to
	// the system and only understood by the binary hosting the plugin and the
	// plugin itself.
	OtherAttributes map[string]string

	// ProvisioningEnabled configures whether provisioning of this plugin is
	// enabled or not. Currently used only in host_path plugin.
	ProvisioningEnabled bool
}

 

2. host_path plugin

    hostPathPlugin 结构体

type hostPathPlugin struct {
	host   volume.VolumeHost
	config volume.VolumeConfig
}

  HostPathVolumeSource结构体  

// Represents a host path mapped into a pod.
// Host path volumes do not support ownership management or SELinux relabeling.
type HostPathVolumeSource struct {
	// Path of the directory on the host.
	// If the path is a symlink, it will follow the link to the real path.
	// More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath
	Path string `json:"path" protobuf:"bytes,1,opt,name=path"`
	// Type for HostPath Volume
	// Defaults to ""
	// More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath
	// +optional
	Type *HostPathType `json:"type,omitempty" protobuf:"bytes,2,opt,name=type"`
}

    type 字段支持以下值：

值	行为
	空字符串（默认）用于向后兼容，这意味着在挂载 hostPath 卷之前不会执行任何检查。
DirectoryOrCreate	如果在给定的路径上没有任何东西存在，那么将根据需要在那里创建一个空目录，权限设置为 0755，与 Kubelet 具有相同的组和所有权。
Directory	路径下必须存在目录
FileOrCreate	如果在给定的路径上没有任何东西存在，那么会根据需要创建一个空文件，权限设置为 0644，与 Kubelet 具有相同的组和所有权。
File	给定的路径下必须存在文件
Socket	路径下必须存在 UNIX 套接字
CharDevice	路径下必须存在字符设备
BlockDevice	路径下必须存在块设备

 

// HostPath volumes represent a bare host file or directory mount.
// The direct at the specified path will be directly exposed to the container.
type hostPath struct {
	path     string
	pathType *v1.HostPathType
	volume.MetricsNil
}

func (hp *hostPath) GetPath() string {
	return hp.path
}

type hostPathMounter struct {
	*hostPath
	readOnly bool
	mounter  mount.Interface
}

   2.1 NewMounter函数

      调用getVolumeSource取得path pathType，填充hostPathMounter实例

func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	hostPathVolumeSource, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	path := hostPathVolumeSource.Path
	pathType := new(v1.HostPathType)
	if hostPathVolumeSource.Type == nil {
		*pathType = v1.HostPathUnset
	} else {
		pathType = hostPathVolumeSource.Type
	}
	return &hostPathMounter{
		hostPath: &hostPath{path: path, pathType: pathType},
		readOnly: readOnly,
		mounter:  plugin.host.GetMounter(plugin.GetPluginName()),
	}, nil
}

  2.2 SetUp函数

     对于hostpath插件SetUp does nothing

// SetUp does nothing.
func (b *hostPathMounter) SetUp(fsGroup *int64) error {
	err := validation.ValidatePathNoBacksteps(b.GetPath())
	if err != nil {
		return fmt.Errorf("invalid HostPath `%s`: %v", b.GetPath(), err)
	}

	if *b.pathType == v1.HostPathUnset {
		return nil
	}
	return checkType(b.GetPath(), b.pathType, b.mounter)
}

  

3. NFS

     路径 pkg/volume/nfs/nfs.go

   nfsPlugin结构体

// This is the primary entrypoint for volume plugins.
// The volumeConfig arg provides the ability to configure recycler behavior.  It is implemented as a pointer to allow nils.
// The nfsPlugin is used to store the volumeConfig and give it, when needed, to the func that creates NFS Recyclers.
// Tests that exercise recycling should not use this func but instead use ProbeRecyclablePlugins() to override default behavior.
func ProbeVolumePlugins(volumeConfig volume.VolumeConfig) []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&nfsPlugin{
			host:   nil,
			config: volumeConfig,
		},
	}
}

type nfsPlugin struct {
	host   volume.VolumeHost
	config volume.VolumeConfig
}

   3.1 Init函数

func (plugin *nfsPlugin) Init(host volume.VolumeHost) error {
	plugin.host = host
	return nil
}

  3.2 NewMounter函数

func (plugin *nfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()))
}

func (plugin *nfsPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) {
	source, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	return &nfsMounter{
		nfs: &nfs{
			volName: spec.Name(),
			mounter: mounter,
			pod:     pod,
			plugin:  plugin,
		},
		server:       source.Server,
		exportPath:   source.Path,
		readOnly:     readOnly,
		mountOptions: util.MountOptionFromSpec(spec),
	}, nil
}

   3.3 SetUp函数

      Mount函数根据初始化kubeletVolumeHost实现了Mounter接口（pkg/kubelet/volume_host.go）

// SetUp attaches the disk and bind mounts to the volume path.
func (b *nfsMounter) SetUp(fsGroup *int64) error {
	return b.SetUpAt(b.GetPath(), fsGroup)
}

func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error {
	notMnt, err := b.mounter.IsNotMountPoint(dir)
	glog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if !notMnt {
		return nil
	}
	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}
	source := fmt.Sprintf("%s:%s", b.server, b.exportPath)
	options := []string{}
	if b.readOnly {
		options = append(options, "ro")
	}
	mountOptions := util.JoinMountOptions(b.mountOptions, options)
	err = b.mounter.Mount(source, dir, "nfs", mountOptions)
	。。。。
	return nil
}

    3.3.1 追踪到使用了mount命令执行

// Mount runs mount(8) using given exec interface.
func (m *execMounter) Mount(source string, target string, fstype string, options []string) error {
	bind, bindRemountOpts := isBind(options)

	if bind {
		err := m.doExecMount(source, target, fstype, []string{"bind"})
		if err != nil {
			return err
		}
		return m.doExecMount(source, target, fstype, bindRemountOpts)
	}

	return m.doExecMount(source, target, fstype, options)
}

// doExecMount calls exec(mount <what> <where>) using given exec interface.
func (m *execMounter) doExecMount(source, target, fstype string, options []string) error {
	glog.V(5).Infof("Exec Mounting %s %s %s %v", source, target, fstype, options)
	mountArgs := makeMountArgs(source, target, fstype, options)
	output, err := m.exec.Run("mount", mountArgs...)
	glog.V(5).Infof("Exec mounted %v: %v: %s", mountArgs, err, string(output))
	if err != nil {
		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n",
			err, "mount", source, target, fstype, options, string(output))
	}

	return err
}