kubernetes 挂载ceph rbd源码解析

kubernetes对于第三方存储本身没有管理，只是使用存储，先看一下

首先是存储插件的初始化，在这个方法里面，通过Init方法初始化

func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, host VolumeHost) error {
    pm.mutex.Lock()
    defer pm.mutex.Unlock()

    if pm.plugins == nil {
        pm.plugins = map[string]VolumePlugin{}
    }

    allErrs := []error{}
    for _, plugin := range plugins {
        name := plugin.GetPluginName()
        if errs := validation.IsQualifiedName(name); len(errs) != 0 {
            allErrs = append(allErrs, fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")))
            continue
        }

        if _, found := pm.plugins[name]; found {
            allErrs = append(allErrs, fmt.Errorf("volume plugin %q was registered more than once", name))
            continue
        }
        err := plugin.Init(host)
        if err != nil {
            glog.Errorf("Failed to load volume plugin %s, error: %s", plugin, err.Error())
            allErrs = append(allErrs, err)
            continue
        }
        pm.plugins[name] = plugin
        glog.V(1).Infof("Loaded volume plugin %q", name)
    }
    return utilerrors.NewAggregate(allErrs)
}

进入rbd的init方法

func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
    plugin.host = host
    return nil
}

存储挂载

func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *types.UnixGroupID) error {
    globalPDPath := manager.MakeGlobalPDName(*b.rbd)
    // TODO: handle failed mounts here.
    notMnt, err := mounter.IsLikelyNotMountPoint(volPath)

    if err != nil && !os.IsNotExist(err) {
        glog.Errorf("cannot validate mountpoint: %s", volPath)
        return err
    }
    if !notMnt {
        return nil
    }
    if err := manager.AttachDisk(b); err != nil {
        glog.Errorf("failed to attach disk")
        return err
    }

    if err := os.MkdirAll(volPath, 0750); err != nil {
        glog.Errorf("failed to mkdir:%s", volPath)
        return err
    }
    // Perform a bind mount to the full path to allow duplicate mounts of the same disk.
    options := []string{"bind"}
    if (&b).GetAttributes().ReadOnly {
        options = append(options, "ro")
    }
    mountOptions := volume.JoinMountOptions(b.mountOptions, options)
    err = mounter.Mount(globalPDPath, volPath, "", mountOptions)
    if err != nil {
        glog.Errorf("failed to bind mount:%s", globalPDPath)
        return err
    }

    if !b.ReadOnly {
        volume.SetVolumeOwnership(&b, fsGroup)
    }

    return nil
}

先是AttachDisk然后mounter.Mount
先看AttachDisk

func (util *RBDUtil) AttachDisk(b rbdMounter) error {
    var err error
    var output []byte

    // create mount point
    globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
    notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
    // in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
    if err != nil && !os.IsNotExist(err) {
        return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
    }
    if !notMnt {
        return nil
    }
    if err = os.MkdirAll(globalPDPath, 0750); err != nil {
        return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
    }

    devicePath, found := waitForPath(b.Pool, b.Image, 1)
    if !found {
        // modprobe
        _, err = b.plugin.execCommand("modprobe", []string{"rbd"})
        if err != nil {
            return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err)
        }

        // fence off other mappers
        if err = util.fencing(b); err != nil {
            return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
        }
        // rbd lock remove needs ceph and image config
        // but kubelet doesn't get them from apiserver during teardown
        // so persit rbd config so upon disk detach, rbd lock can be removed
        // since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
        // the json file remains invisible during rbd mount and thus won't be removed accidentally.
        util.persistRBD(b, globalPDPath)

        // rbd map
        l := len(b.Mon)
        // avoid mount storm, pick a host randomly
        start := rand.Int() % l
        // iterate all hosts until mount succeeds.
        for i := start; i < start+l; i++ {
            mon := b.Mon[i%l]
            glog.V(1).Infof("rbd: map mon %s", mon)
            if b.Secret != "" {
                output, err = b.plugin.execCommand("rbd",
                    []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key=" + b.Secret})
            } else {
                output, err = b.plugin.execCommand("rbd",
                    []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring})
            }
            if err == nil {
                break
            }
            glog.V(1).Infof("rbd: map error %v %s", err, string(output))
        }
        if err != nil {
            return fmt.Errorf("rbd: map failed %v %s", err, string(output))
        }
        devicePath, found = waitForPath(b.Pool, b.Image, 10)
        if !found {
            return errors.New("Could not map image: Timeout after 10s")
        }
    }

    // mount it
    if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
        err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
    }
    return err
}

上面只是挂着到主机上面，mounter.Mount挂着到容器里面

func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
    // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
    // All Linux distros are expected to be shipped with a mount utility that an support bind mounts.
    mounterPath := ""
    bind, bindRemountOpts := isBind(options)
    if bind {
        err := doMount(mounterPath, defaultMountCommand, source, target, fstype, []string{"bind"})
        if err != nil {
            return err
        }
        return doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts)
    }
    // The list of filesystems that require containerized mounter on GCI image cluster
    fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs")
    if fsTypesNeedMounter.Has(fstype) {
        mounterPath = mounter.mounterPath
    }
    return doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
}

效果如下：

mount |grep rbd0

/dev/rbd0 on /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/tenx-pool-image-qinzhao.CID-ca4135da3326.datadir-zookeeper-2 type ext4 (rw,relatime,stripe=1024,data=ordered)

/dev/rbd0 on /var/lib/kubelet/pods/d03c7a87-4b3c-11e7-9b5e-5254eec04736/volumes/kubernetes.io~rbd/qinzhao.datadir-zookeeper-2 type ext4 (rw,relatime,stripe=1024,data=ordered)

上面是挂载到主机，下面是挂载到容器。