local-path-provisioner 源码解析

程序入口main.go

main 函数

日志模块：

logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true})

配置命令行应用：

a := cli.NewApp()
a.Version = VERSION #设置help显示版本号为0.0.1
a.Usage = "Local Path Provisioner" #配置help显示usage内容
a.Before = func(c *cli.Context) error { #如果参数debug为true 设置启动模式为debug模式
   if c.GlobalBool("debug") {
      logrus.SetLevel(logrus.DebugLevel)
   }
   return nil
}
a.Flags = []cli.Flag{ #是否开启debug参数--debug或者-d
   cli.BoolFlag{
      Name:   "debug, d",
      Usage:  "enable debug logging level",
      EnvVar: "RANCHER_DEBUG",
   },
}
a.Commands = []cli.Command{ #程序执行入口命令，执行命令列表
   StartCmd(),
}
a.CommandNotFound = cmdNotFound #如果找不到这个命令执行cmdNotFound函数
a.OnUsageError = onUsageError   #如果使用错误执行onUsageError函数
if err := a.Run(os.Args); err != nil { #接收os.args参数启动程序
   logrus.Fatalf("Critical error: %v", err)
}

 

StartCmd函数(用于配置参数)：

func StartCmd() cli.Command {
   return cli.Command{
      Name: "start",  #配置第一个参数为start的参数列表
      Flags: []cli.Flag{
         cli.StringFlag{
            Name:  FlagConfigFile,  #配置--config参数
            Usage: "Required. Provisioner configuration file.",
            Value: "",
         },
         cli.StringFlag{
            Name:   FlagProvisionerName,  #配置--provisioner-name参数 默认值rancher.io/local-path
            Usage:  "Required. Specify Provisioner name.",
            EnvVar: EnvProvisionerName,
            Value:  DefaultProvisionerName,
         },
         cli.StringFlag{
            Name:   FlagNamespace,  #配置--FlagNamespace参数 默认值为local-path-storage，
            Usage:  "Required. The namespace that Provisioner is running in",
            EnvVar: EnvNamespace,
            Value:  DefaultNamespace,
         },
         cli.StringFlag{
            Name:   FlagHelperImage,  #配置--helper-image参数 默认值为busybox 用于创建和删除目录
            Usage:  "Required. The helper image used for create/delete directories on the host",
            EnvVar: EnvHelperImage,
            Value:  DefaultHelperImage,
         },
      },
      Action: func(c *cli.Context) { #执行的动作 执行startDaemon函数
         if err := startDaemon(c); err != nil {
            logrus.Fatalf("Error starting daemon: %v", err)
         }
      },
   }
}

startDaemon函数:

stopCh := make(chan struct{}) #设置一个空结构体 stopCh
RegisterShutdownChannel(stopCh) #执行RegisterShutdownChannel函数，

RegisterShutdownChannel函数：

func RegisterShutdownChannel(done chan struct{}) {该函数的作用就是接收信号值，只接收syscall.SIGINT(2)，syscall.SIGTERM(15)，
   sigs := make(chan os.Signal, 1)
   signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
   go func() {
      sig := <-sigs  #等待接收sigs 如果没有收到一直处于该阶段 不往下执行
      logrus.Infof("Receive %v to exit", sig)
      close(done) #关闭该程序
   }()
}

//通过集群内部配置创建 k8s 配置信息，通过 KUBERNETES_SERVICE_HOST 和 KUBERNETES_SERVICE_PORT 环境变量方式获取
//默认tokenfile  rootCAFile 在/var/run/secrets/kubernetes.io/serviceaccount/
config, err := rest.InClusterConfig()
if err != nil {
   return errors.Wrap(err, "unable to get client config")
}

kubeClient, err := clientset.NewForConfig(config)
if err != nil {
   return errors.Wrap(err, "unable to get k8s client")
}

//获取k8s版本号
serverVersion, err := kubeClient.Discovery().ServerVersion()
if err != nil {
   return errors.Wrap(err, "Cannot start Provisioner: failed to get Kubernetes server version")
}
//以下获取参数传递的值
configFile := c.String(FlagConfigFile)
if configFile == "" {
   return fmt.Errorf("invalid empty flag %v", FlagConfigFile)
}
provisionerName := c.String(FlagProvisionerName)
if provisionerName == "" {
   return fmt.Errorf("invalid empty flag %v", FlagProvisionerName)
}
namespace := c.String(FlagNamespace)
if namespace == "" {
   return fmt.Errorf("invalid empty flag %v", FlagNamespace)
}
helperImage := c.String(FlagHelperImage)
if helperImage == "" {
   return fmt.Errorf("invalid empty flag %v", FlagHelperImage)
}
//获取参数之后执行NewProvisioner函数
provisioner, err := NewProvisioner(stopCh, kubeClient, configFile, namespace, helperImage)
if err != nil {
   return err
}

NewProvisioner函数：

func NewProvisioner(stopCh chan struct{}, kubeClient *clientset.Clientset, configFile, namespace, helperImage string) (*LocalPathProvisioner, error) {
   //初始化LocalPathProvisioner结构体
   p := &LocalPathProvisioner{
      stopCh: stopCh,

      kubeClient:  kubeClient,
      namespace:   namespace,
      helperImage: helperImage,

      // config will be updated shortly by p.refreshConfig()
      config:      nil,
      configFile:  configFile,
      configData:  nil,
      configMutex: &sync.RWMutex{},
   }
   //刷新配置
   if err := p.refreshConfig(); err != nil {
      return nil, err
   }
   //监听配置变化并刷新
   p.watchAndRefreshConfig()
   return p, nil
}

refreshConfig函数：重载配置函数 加载config文件并加载到内存里

func (p *LocalPathProvisioner) refreshConfig() error {
   p.configMutex.Lock()  #写锁定
   defer p.configMutex.Unlock()  #执行完成之后解锁

   configData, err := loadConfigFile(p.configFile) #加载配置文件，解析成json格式
   if err != nil {
      return err
   }
   // no need to update
   if reflect.DeepEqual(configData, p.configData) { #对比新的配置和旧的配置 如果一样直接返回nil
      return nil
   }
   config, err := canonicalizeConfig(configData) #获取配置,设置NodePathMap的值
   if err != nil {
      return err
   }
   // only update the config if the new config file is valid
   p.configData = configData
   p.config = config

   output, err := json.Marshal(p.configData)
   if err != nil {
      return err
   }
   logrus.Debugf("Applied config: %v", string(output))

   return err
}

watchAndRefreshConfig 函数： 该函数是一个死循环 并且每30s执行重载配置函数

func (p *LocalPathProvisioner) watchAndRefreshConfig() {
   go func() {
      for {
         select {
         case <-time.Tick(ConfigFileCheckInterval): #设置配置检查间隔30s
            if err := p.refreshConfig(); err != nil { #刷新配置
               logrus.Errorf("failed to load the new config file: %v", err)
            }
         case <-p.stopCh: #等待stopCh 接收停止信号
            logrus.Infof("stop watching config file")
            return
         }
      }
   }()
}

 创建一个新的Provision控制器（主要用来生成新的Provision控制器，实例化 ProvisionController 结构，主要是为 PVC 提供 PV 操作，并初始化pv pvc storageclass的informers，当创建或者删除或者更新pv pvc storageclass时会触发enqueueClaim函数，该函数主要作用就是将pvc加入到workqueue队列，Controller 收到 WorkQueue 中的事件会根据对应的类型触发对应的回调函数）：

 

pc := pvController.NewProvisionController(
   kubeClient,
   provisionerName,
   provisioner,
   serverVersion.GitVersion,

 NewProvisionController函数:

func NewProvisionController(
   client kubernetes.Interface,
   provisionerName string,
   provisioner Provisioner,
   kubeVersion string,
   options ...func(*ProvisionController) error,
) *ProvisionController {
   id, err := os.Hostname()   //获取主机名
   if err != nil {
      glog.Fatalf("Error getting hostname: %v", err)
   }
   // add a uniquifier so that two processes on the same host don't accidentally both become active
   id = id + "_" + string(uuid.NewUUID())  //生成一个随机id号
   component := provisionerName + "_" + id 

   v1.AddToScheme(scheme.Scheme)
   broadcaster := record.NewBroadcaster() //event的初始化
   broadcaster.StartLogging(glog.Infof)
   broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)})
   eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: component})

   controller := &ProvisionController{
      client:                    client,
      provisionerName:           provisionerName,
      provisioner:               provisioner,
      kubeVersion:               utilversion.MustParseSemantic(kubeVersion),
      id:                        id,
      component:                 component,
      eventRecorder:             eventRecorder,
      resyncPeriod:              DefaultResyncPeriod,
      exponentialBackOffOnError: DefaultExponentialBackOffOnError,
      threadiness:               DefaultThreadiness,
      failedProvisionThreshold:  DefaultFailedProvisionThreshold,
      failedDeleteThreshold:     DefaultFailedDeleteThreshold,
      leaderElection:            DefaultLeaderElection,
      leaderElectionNamespace:   getInClusterNamespace(),
      leaseDuration:             DefaultLeaseDuration,
      renewDeadline:             DefaultRenewDeadline,
      retryPeriod:               DefaultRetryPeriod,
      metricsPort:               DefaultMetricsPort,
      metricsAddress:            DefaultMetricsAddress,
      metricsPath:               DefaultMetricsPath,
      addFinalizer:              DefaultAddFinalizer,
      hasRun:                    false,
      hasRunLock:                &sync.Mutex{},
   }

   for _, option := range options {
      err := option(controller)
      if err != nil {
         glog.Fatalf("Error processing controller options: %s", err)
      }
   }

   var rateLimiter workqueue.RateLimiter
   if controller.rateLimiter != nil {
      // rateLimiter set via parameter takes precedence
      rateLimiter = controller.rateLimiter
   } else if controller.exponentialBackOffOnError {
      rateLimiter = workqueue.NewMaxOfRateLimiter(
         workqueue.NewItemExponentialFailureRateLimiter(15*time.Second, 1000*time.Second),
         &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
      )
   } else {
      rateLimiter = workqueue.NewMaxOfRateLimiter(
         workqueue.NewItemExponentialFailureRateLimiter(15*time.Second, 15*time.Second),
         &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
      )
   }
   controller.claimQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "claims")
   controller.volumeQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "volumes")

   informer := informers.NewSharedInformerFactory(client, controller.resyncPeriod)

   // ----------------------
   // PersistentVolumeClaims

   claimHandler := cache.ResourceEventHandlerFuncs{
      AddFunc:    func(obj interface{}) { controller.enqueueClaim(obj) },
      UpdateFunc: func(oldObj, newObj interface{}) { controller.enqueueClaim(newObj) },
      DeleteFunc: func(obj interface{}) {
         // NOOP. The claim is either in claimsInProgress and in the queue, so it will be processed as usual
         // or it's not in claimsInProgress and then we don't care
      },
   }

   if controller.claimInformer != nil {
      controller.claimInformer.AddEventHandlerWithResyncPeriod(claimHandler, controller.resyncPeriod)
   } else {
      controller.claimInformer = informer.Core().V1().PersistentVolumeClaims().Informer()
      controller.claimInformer.AddEventHandler(claimHandler)
   }
   controller.claimInformer.AddIndexers(cache.Indexers{uidIndex: func(obj interface{}) ([]string, error) {
      uid, err := getObjectUID(obj)
      if err != nil {
         return nil, err
      }
      return []string{uid}, nil
   }})
   controller.claimsIndexer = controller.claimInformer.GetIndexer()

   // -----------------
   // PersistentVolumes

   volumeHandler := cache.ResourceEventHandlerFuncs{
      AddFunc:    func(obj interface{}) { controller.enqueueVolume(obj) },
      UpdateFunc: func(oldObj, newObj interface{}) { controller.enqueueVolume(newObj) },
      DeleteFunc: func(obj interface{}) { controller.forgetVolume(obj) },
   }

   if controller.volumeInformer != nil {
      controller.volumeInformer.AddEventHandlerWithResyncPeriod(volumeHandler, controller.resyncPeriod)
   } else {
      controller.volumeInformer = informer.Core().V1().PersistentVolumes().Informer()
      controller.volumeInformer.AddEventHandler(volumeHandler)
   }
   controller.volumes = controller.volumeInformer.GetStore()

   // --------------
   // StorageClasses

   // no resource event handler needed for StorageClasses
   if controller.classInformer == nil {
      if controller.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) {
         controller.classInformer = informer.Storage().V1().StorageClasses().Informer()
      } else {
         controller.classInformer = informer.Storage().V1beta1().StorageClasses().Informer()
      }
   }
   controller.classes = controller.classInformer.GetStore()

   if controller.createProvisionerPVLimiter != nil {
      glog.V(2).Infof("Using saving PVs to API server in background")
      controller.volumeStore = NewVolumeStoreQueue(client, controller.createProvisionerPVLimiter, controller.claimsIndexer, controller.eventRecorder)
   } else {
      if controller.createProvisionedPVBackoff == nil {
         // Use linear backoff with createProvisionedPVInterval and createProvisionedPVRetryCount by default.
         if controller.createProvisionedPVInterval == 0 {
            controller.createProvisionedPVInterval = DefaultCreateProvisionedPVInterval
         }
         if controller.createProvisionedPVRetryCount == 0 {
            controller.createProvisionedPVRetryCount = DefaultCreateProvisionedPVRetryCount
         }
         controller.createProvisionedPVBackoff = &wait.Backoff{
            Duration: controller.createProvisionedPVInterval,
            Factor:   1, // linear backoff
            Steps:    controller.createProvisionedPVRetryCount,
            //Cap:      controller.createProvisionedPVInterval,
         }
      }
      glog.V(2).Infof("Using blocking saving PVs to API server")
      controller.volumeStore = NewBackoffStore(client, controller.eventRecorder, controller.createProvisionedPVBackoff, controller)
   }

   return controller
}

pc.Run(stopCh)启动该服务

 

run函数：先进行选主，leaderElection选主运行(考虑到多副本情况下)，选主规则是根据endpoints(除了endpoints之外还有configmap)， 主要的作用就是当设置了metricsPort 会启动一个metrics端口用于promethues收集监控数据。之后启动三个informer实例 分别是pvc pv stroageclass 

 

 

// Run starts all of this controller's control loops
func (ctrl *ProvisionController) Run(_ <-chan struct{}) {
   // TODO: arg is as of 1.12 unused. Nothing can ever be cancelled. Should
   // accept a context instead and use it instead of context.TODO(), but would
   // break API. Not urgent: realistically, users are simply passing in
   // wait.NeverStop() anyway.

   run := func(ctx context.Context) {
      glog.Infof("Starting provisioner controller %s!", ctrl.component)
      defer utilruntime.HandleCrash()
      defer ctrl.claimQueue.ShutDown()
      defer ctrl.volumeQueue.ShutDown()

      ctrl.hasRunLock.Lock()
      ctrl.hasRun = true
      ctrl.hasRunLock.Unlock()
      if ctrl.metricsPort > 0 { 。  //如果metricsPort大于0 那么就上报数据给promethues 主要用来监控数据
         prometheus.MustRegister([]prometheus.Collector{
            metrics.PersistentVolumeClaimProvisionTotal,
            metrics.PersistentVolumeClaimProvisionFailedTotal,
            metrics.PersistentVolumeClaimProvisionDurationSeconds,
            metrics.PersistentVolumeDeleteTotal,
            metrics.PersistentVolumeDeleteFailedTotal,
            metrics.PersistentVolumeDeleteDurationSeconds,
         }...)
         http.Handle(ctrl.metricsPath, promhttp.Handler())
         address := net.JoinHostPort(ctrl.metricsAddress, strconv.FormatInt(int64(ctrl.metricsPort), 10))
         glog.Infof("Starting metrics server at %s\n", address)
         go wait.Forever(func() {
            err := http.ListenAndServe(address, nil)
            if err != nil {
               glog.Errorf("Failed to listen on %s: %v", address, err)
            }
         }, 5*time.Second)
      }

      // If a external SharedInformer has been passed in, this controller
      // should not call Run again
      if !ctrl.customClaimInformer {
         go ctrl.claimInformer.Run(ctx.Done())
      }
      if !ctrl.customVolumeInformer {
         go ctrl.volumeInformer.Run(ctx.Done())
      }
      if !ctrl.customClassInformer {
         go ctrl.classInformer.Run(ctx.Done())
      }
        //从apiserver同步资源
      if !cache.WaitForCacheSync(ctx.Done(), ctrl.claimInformer.HasSynced, ctrl.volumeInformer.HasSynced, ctrl.classInformer.HasSynced) {
         return
      }
        //启动线程工作 默认4个并发(重点，用于处理pvc，pv的工作队列volumeQueue，claimQueue)
      for i := 0; i < ctrl.threadiness; i++ {
         go wait.Until(ctrl.runClaimWorker, time.Second, context.TODO().Done())
         go wait.Until(ctrl.runVolumeWorker, time.Second, context.TODO().Done())
      }

      glog.Infof("Started provisioner controller %s!", ctrl.component)

      select {}
   }

   go ctrl.volumeStore.Run(context.TODO(), DefaultThreadiness)

   if ctrl.leaderElection { //创建一个选主锁
      rl, err := resourcelock.New("endpoints",
         ctrl.leaderElectionNamespace,
         strings.Replace(ctrl.provisionerName, "/", "-", -1), //provisionerName的默认值是rancher.io/local-path
         ctrl.client.CoreV1(),
         nil,
         resourcelock.ResourceLockConfig{
            Identity:      ctrl.id,
            EventRecorder: ctrl.eventRecorder,
         })
      if err != nil {
         glog.Fatalf("Error creating lock: %v", err)
      }

      leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
         Lock:          rl,
         LeaseDuration: ctrl.leaseDuration,
         RenewDeadline: ctrl.renewDeadline,
         RetryPeriod:   ctrl.retryPeriod,
         Callbacks: leaderelection.LeaderCallbacks{
            OnStartedLeading: run,
            OnStoppedLeading: func() {
               glog.Fatalf("leaderelection lost")
            },
         },
      })
      panic("unreachable")
   } else {
      run(context.TODO())
   }
}

工作流程：

runClaimWorker -> processNextClaimWorkItem -> syncClaimHandler -> syncClaim -> provisionClaimOperation ->Provision

runVolumeWorker -> processNextVolumeWorkItem -> syncVolumeHandler -> syncVolume ->deleteVolumeOperation

 

runClaimWorker工作流程:

processNextClaimWorkItem：判断pvc是否完成创建

syncClaimHandler： 从informer的缓存中获取对象，然后调用syncClaim

syncClaim： 通过shouldProvision判断是否需要生成pv ，kubernetes版本大于1.5 如果pvc的Annotations存在"volume.beta.kubernetes.io/storage-provisioner"，并且provisionerName的值rancher.io/local-path 返回true

如果是1.4版本以下的Annotations v1.BetaStorageClassAnnotation 并且StorageClassName 存在 并且可以通过getStorageClass获取到StorageClass

如果返回true 执行provisionClaimOperation函数

provisionClaimOperation： 主要是执行Provision函数，该函数需要自己编写 如下

，主要实现创建pvc以及通过创建pod挂载对应目录之后创建对应目录，最后将PersistentVolume对象返回provisionClaimOperation函数，之后绑定pv与pvc，通过绑定ClaimRef值,整体逻辑就是先创建pvc之后通过创建pod在对应node下创建文件夹之后创建pv。

func (p *LocalPathProvisioner) Provision(opts pvController.ProvisionOptions) (*v1.PersistentVolume, error) {
   pvc := opts.PVC
   if pvc.Spec.Selector != nil {
      return nil, fmt.Errorf("claim.Spec.Selector is not supported")
   }
   for _, accessMode := range pvc.Spec.AccessModes {
      if accessMode != v1.ReadWriteOnce {
         return nil, fmt.Errorf("Only support ReadWriteOnce access mode")
      }
   }
   node := opts.SelectedNode
   if opts.SelectedNode == nil {
      return nil, fmt.Errorf("configuration error, no node was specified")
   }

   basePath, err := p.getRandomPathOnNode(node.Name)
   if err != nil {
      return nil, err
   }

   name := opts.PVName
   folderName := strings.Join([]string{name, opts.PVC.Namespace, opts.PVC.Name}, "_")

   path := filepath.Join(basePath, folderName)
   logrus.Infof("Creating volume %v at %v:%v", name, node.Name, path)

   createCmdsForPath := []string{
      "mkdir",
      "-m", "0777",
      "-p",
   }
   if err := p.createHelperPod(ActionTypeCreate, createCmdsForPath, name, path, node.Name); err != nil {
      return nil, err
   }

   fs := v1.PersistentVolumeFilesystem
   hostPathType := v1.HostPathDirectoryOrCreate
   return &v1.PersistentVolume{
      ObjectMeta: metav1.ObjectMeta{
         Name: name,
      },
      Spec: v1.PersistentVolumeSpec{
         PersistentVolumeReclaimPolicy: *opts.StorageClass.ReclaimPolicy,
         AccessModes:                   pvc.Spec.AccessModes,
         VolumeMode:                    &fs,
         Capacity: v1.ResourceList{
            v1.ResourceName(v1.ResourceStorage): pvc.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
         },
         PersistentVolumeSource: v1.PersistentVolumeSource{
            HostPath: &v1.HostPathVolumeSource{
               Path: path,
               Type: &hostPathType,
            },
         },
         NodeAffinity: &v1.VolumeNodeAffinity{
            Required: &v1.NodeSelector{
               NodeSelectorTerms: []v1.NodeSelectorTerm{
                  {
                     MatchExpressions: []v1.NodeSelectorRequirement{
                        {
                           Key:      KeyNode,
                           Operator: v1.NodeSelectorOpIn,
                           Values: []string{
                              node.Name,
                           },
                        },
                     },
                  },
               },
            },
         },
      },
   }, nil
}

从这里面我们可以知道如何在在不同节点 设置不同的volume目录。我们只需要在配置文件里设置node对应的path就可以里