【kubernetes/k8s源码分析】 k8s local storage 源码分析

    Local Storage 就是 Kubernetes 集群中每个节点的本地存储，在 Kubernetes 1.7 中 kubelet 可以支持对 kube-reserved 和system-reserved 指定本地存储资源。Local 存储是为了替换 hostPath volume

   kubernetes v1.14.0 正式发布，新特性就是本地持久化管理( Local StorageManagement)特性正式 GA(稳定)

 

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
# Supported policies: Delete, Retain
reclaimPolicy: Delete

 

apiVersion: v1
kind: PersistentVolume
metadata:
  name: example-local-pv
spec:
  capacity:
    storage: 5Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /tmp/aaaa
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - master-node

 

1. NewMounter 函数

    实例化 localVolumeMounter，熟悉的套路

func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
	_, readOnly, err := getVolumeSource(spec)
	if err != nil {
		return nil, err
	}

	globalLocalPath, err := plugin.getGlobalLocalPath(spec)
	if err != nil {
		return nil, err
	}

	return &localVolumeMounter{
		localVolume: &localVolume{
			pod:             pod,
			podUID:          pod.UID,
			volName:         spec.Name(),
			mounter:         plugin.host.GetMounter(plugin.GetPluginName()),
			plugin:          plugin,
			globalPath:      globalLocalPath,
			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(pod.UID, stringsutil.EscapeQualifiedNameForDisk(localVolumePluginName), spec.Name())),
		},
		mountOptions: util.MountOptionFromSpec(spec),
		readOnly:     readOnly,
	}, nil

}

 

2. SetUp

    2.1 global path 不能为空

m.plugin.volumeLocks.LockKey(m.globalPath)
defer m.plugin.volumeLocks.UnlockKey(m.globalPath)

if m.globalPath == "" {
	return fmt.Errorf("LocalVolume volume %q path is empty", m.volName)
}

    2.2 IsNotMountPoint 函数查验是否已经mount了

notMnt, err := m.mounter.IsNotMountPoint(dir)
klog.V(4).Infof("LocalVolume mount setup: PodDir(%s) VolDir(%s) Mounted(%t) Error(%v), ReadOnly(%t)", dir, m.globalPath, !notMnt, err, m.readOnly)
if err != nil && !os.IsNotExist(err) {
	klog.Errorf("cannot validate mount point: %s %v", dir, err)
	return err
}

if !notMnt {
	return nil
}

    2.3 mount 操作

      将 /var/lib/kubelet/pods/fc695a36-8475-11e9-a5a0-0800271c9f15/volumes/kubernetes.io~local-volume/example-local-pv 挂载在容器的目录

err = m.mounter.Mount(globalPath, dir, "", mountOptions)
if err != nil {
	klog.Errorf("Mount of volume %s failed: %v", dir, err)
	notMnt, mntErr := m.mounter.IsNotMountPoint(dir)
	if mntErr != nil {
		klog.Errorf("IsNotMountPoint check failed: %v", mntErr)
		return err
	}

   

   注意： 

        需要运行节点上面去创建本地目录，否则POD将会启动失败

Events:
  Type     Reason       Age                  From                  Message
  ----     ------       ----                 ----                  -------
  Normal   Scheduled    2m30s                default-scheduler     Successfully assigned default/example-pod to master-node
  Warning  FailedMount  86s (x8 over 2m30s)  kubelet, master-node  MountVolume.NewMounter initialization failed for volume "example-local-pv" : path "/tmp/aaaa" does not exist