kubernetes添加第三方存储(三)

前面两篇文章已经介绍了整体的内容，下面就已对青云的存储为例，先提供一个Provision的实现

func (c *volumeProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
    glog.V(4).Infof("qingcloudVolumeProvisioner Provision called, options: [%+v]", options)

    // TODO: implement PVC.Selector parsing
    if options.PVC.Spec.Selector != nil {
        return nil, fmt.Errorf("claim.Spec.Selector is not supported for dynamic provisioning on qingcloud")
    }

    // Validate access modes
    found := false
    for _, mode := range options.PVC.Spec.AccessModes {
        if mode == v1.ReadWriteOnce {
            found = true
        }
    }
    if !found {
        return nil, fmt.Errorf("Qingcloud volume only supports ReadWriteOnce mounts")
    }

    volumeOptions := &VolumeOptions{}

    hasSetType := false
    for k, v := range options.Parameters {
        switch strings.ToLower(k) {
        case "type":
            if !supportVolumeTypes.Has(v) {
                return nil, fmt.Errorf("invalid option '%q' for qingcloud-volume-provisioner, it only can be 0, 2, 3",
                    k)
            }
            volumeTypeInt, _ := strconv.Atoi(v)
            volumeOptions.VolumeType = VolumeType(volumeTypeInt)
            hasSetType = true
        default:
            return nil, fmt.Errorf("invalid option '%q' for qingcloud-volume-provisioner", k)
        }
    }

    //auto set volume type by instance class.
    if !hasSetType {
        volumeOptions.VolumeType = c.manager.GetDefaultVolumeType()
    }

    capacity := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
    sizeGB, err := RoundUpVolumeCapacity(capacity, volumeOptions.VolumeType)
    if err != nil {
        return nil, err
    }
    volumeOptions.CapacityGB = sizeGB

    //use pv name as volumeName
    volumeOptions.VolumeName = options.PVName
    volumeID, err := c.manager.CreateVolume(volumeOptions)
    if err != nil {
        glog.V(2).Infof("Error creating qingcloud volume: %v", err)
        return nil, err
    }
    glog.V(2).Infof("Successfully created qingcloud volume %s", volumeID)

    storageClassName := ""
    if options.PVC.Spec.StorageClassName != nil {
        storageClassName = *options.PVC.Spec.StorageClassName
    }

    annotations := make(map[string]string)
    annotations[annCreatedBy] = createdBy
    annotations[annProvisionerId] = ProvisionerName

    flexVolumeConfig := make(map[string]string)
    flexVolumeConfig[OptionVolumeID] = volumeID

    fsType ,ok := options.PVC.ObjectMeta.Annotations["kubernetes.io/fsType"]
    if !ok{
        fsType = DefaultFSType
    }
    pv := &v1.PersistentVolume{
        ObjectMeta: metav1.ObjectMeta{
            Name:        options.PVName,
            Labels:      map[string]string{},
            Annotations: annotations,
        },
        Spec: v1.PersistentVolumeSpec{
            PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
            AccessModes:                   []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
            Capacity: v1.ResourceList{
                v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
            },
            StorageClassName: storageClassName,
            PersistentVolumeSource: v1.PersistentVolumeSource{
                FlexVolume: &v1.FlexVolumeSource{
                    Driver:   FlexDriverName,
                    FSType:   fsType,
                    ReadOnly: false,
                    Options:  flexVolumeConfig,
                },
            },
        },
    }

    return pv, nil
}

上面的代码就是创建pv的整个过程

然后是删除的方法

func (c *volumeProvisioner) Delete(volume *v1.PersistentVolume) error {
    if volume.Name == "" {
        return fmt.Errorf("volume name cannot be empty %#v", volume)
    }

    if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain {
        if volume.Spec.PersistentVolumeSource.FlexVolume == nil {
            return fmt.Errorf("volume [%s] not support by qingcloud-volume-provisioner", volume.Name)
        }
        volumeID := volume.Spec.PersistentVolumeSource.FlexVolume.Options["volumeID"]
        if volumeID == "" {
            return fmt.Errorf("Spec.PersistentVolumeSource.FlexVolume.Options[\"volumeID\"]  cannot be empty %#v", volume)
        }
        _, err := c.manager.DeleteVolume(volumeID)
        if err != nil {
            return err
        }
        return nil
    }

    return nil
}

然后是二进制执行文件

func handler(op string, args []string) flex.VolumeResult {
    volumePlugin, err := qingcloud.NewFlexVolumePlugin()

    if err != nil {
        return flex.NewVolumeError("Error init FlexVolumePlugin")
    }

    var ret flex.VolumeResult

    switch op {
    case "init":
        ret = volumePlugin.Init()
    case "attach":
        if len(args) < 2 {
            return flex.NewVolumeError("attach requires options in json format and a node name")
        }
        ret = volumePlugin.Attach(ensureVolumeOptions(args[0]), args[1])
    case "isattached":
        if len(args) < 2 {
            return flex.NewVolumeError("isattached requires options in json format and a node name")
        }
        ret = volumePlugin.IsAttached(ensureVolumeOptions(args[0]), args[1])
    case "detach":
        if len(args) < 2 {
            return flex.NewVolumeError("detach requires a device path and a node name")
        }
        ret = volumePlugin.Detach(args[0], args[1])
    case "mountdevice":
        if len(args) < 3 {
            return flex.NewVolumeError("mountdevice requires a mount path, a device path and mount options")
        }
        ret = volumePlugin.MountDevice(args[0], args[1], ensureVolumeOptions(args[2]))
    case "unmountdevice":
        if len(args) < 1 {
            return flex.NewVolumeError("unmountdevice requires a mount path")
        }
        ret = volumePlugin.UnmountDevice(args[0])
    case "waitforattach":
        if len(args) < 2 {
            return flex.NewVolumeError("waitforattach requires a device path and options in json format")
        }
        ret = volumePlugin.WaitForAttach(args[0], ensureVolumeOptions(args[1]))
    case "getvolumename":
        if len(args) < 1 {
            return flex.NewVolumeError("getvolumename requires options in json format")
        }
        ret = volumePlugin.GetVolumeName(ensureVolumeOptions(args[0]))
    default:
        ret = flex.NewVolumeNotSupported(op)
    }
    return ret
}

实现了attach、MountDevice等方法。具体以一个attached为例

func (p *flexVolumePlugin) Attach(options flex.VolumeOptions, node string) flex.VolumeResult {
    glog.V(4).Infof("Attach")
    volumeID, _ := options[OptionVolumeID].(string)
    pvOrVolumeName, _ := options[OptionPVorVolumeName].(string)
    // flexVolumeDriver GetVolumeName is not yet supported,  so PVorVolumeName is pvName, and store pvName to volumeName
    if !isVolumeID(pvOrVolumeName) {
        err := p.manager.UpdateVolume(volumeID, pvOrVolumeName)
        if err != nil {
            return flex.NewVolumeError("Error updating volume (%s) name to (%s) : %s", volumeID, pvOrVolumeName, err.Error())
        }
    }
    // VolumeManager.AttachVolume checks if disk is already attached to node and
    // succeeds in that case, so no need to do that separately.
    _, err := p.manager.AttachVolume(volumeID, node)

    if err != nil {
        //ignore already attached error
        if !strings.Contains(err.Error(), "have been already attached to instance") {
            glog.Errorf("Error attaching volume %q: %+v", volumeID, err)
            return flex.NewVolumeError("Error attaching volume %q to node %s: %+v", volumeID, node, err)
        }
    }

    return flex.NewVolumeSuccess().WithDevicePath(volumeID)
}

上面的AttachVolume通过调用青云api完成加载

if !attached {
        output, err := vm.volumeService.AttachVolumes(&qcservice.AttachVolumesInput{
            Volumes:  []*string{&volumeID},
            Instance: &instanceID,
        })
        if err != nil {
            return "", err
        }
        jobID := *output.JobID
        //ignore wait job error
        qcclient.WaitJob(vm.jobService, jobID, operationWaitTimeout, waitInterval)
    }

如果你还有自己的存储，通过相同的方式也就可以介入到k8s了。