【kubernetes/k8s源码分析】 rook operator启动源码分析

最新推荐文章于 2024-06-12 08:56:36 发布

张忠琳

最新推荐文章于 2024-06-12 08:56:36 发布

阅读量2.5k

点赞数 2

分类专栏： # kubernetes CSI 存储

本文链接：https://blog.csdn.net/zhonglinzhang/article/details/89840182

版权

kubernetes CSI 存储专栏收录该内容

34 篇文章 12 订阅

订阅专栏

https://github.com/rook/rook

operator：作用启动监控存储集群，包括mon，osd，mgr等服务，可以支持 block 块存储，file system 文件存储，object 对象存储 (S3/Swift)。 启动包括ceph agent 与 discover服务，agent 包含 Flexvolume plugin，包括attaching network storage devices, mounting volumes, and formating the filesystem 三种操作。

启动命令

Usage:
rook ceph operator [flags]

启动参数

包括的ceph镜像包括attacher，provisioner，csi-node-driver-registrar，cephfsplugin，rbdplugin

包括的配置存入/etc/ceph-csi目录，

--mon-healthcheck-interval：mon健康监测间隔，默认45s
--mon-out-timeout：mon超时时间，默认未10m

--alsologtostderr=false, --csi-attacher-image=quay.io/k8scsi/csi-attacher:v1.0.1,

--csi-cephfs-image=quay.io/cephcsi/cephfsplugin:v1.0.0,

--csi-cephfs-plugin-template-path=/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml,

--csi-cephfs-provisioner-template-path=/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner.yaml,

--csi-enable-cephfs=false, --csi-enable-rbd=false,

--csi-provisioner-image=quay.io/k8scsi/csi-provisioner:v1.0.1,

--csi-rbd-image=quay.io/cephcsi/rbdplugin:v1.0.0,

--csi-rbd-plugin-template-path=/etc/ceph-csi/rbd/csi-rbdplugin.yaml,

--csi-rbd-provisioner-template-path=/etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml,

--csi-registrar-image=quay.io/k8scsi/csi-node-driver-registrar:v1.0.2,

--csi-snapshotter-image=quay.io/k8scsi/csi-snapshotter:v1.0.1,

--help=false, --log-flush-frequency=5s, --log-level=INFO,

--log_backtrace_at=:0, --log_dir=, --log_file=, --logtostderr=true,

--mon-healthcheck-interval=45s, --mon-out-timeout=10m0s,

--skip_headers=false, --stderrthreshold=2, --v=0, --vmodule=

1. startOperator函数

1.1 建立kube apiserver的客户端，比较容易理解

clientset, apiExtClientset, rookClientset, err := rook.GetClientset()
if err != nil {
   rook.TerminateFatal(fmt.Errorf("failed to get k8s client. %+v\n", err))
}

1.2 创建上下文，比如网络信息，路径设置为/var/lib/rook，客户端

context := createContext()
context.NetworkInfo = clusterd.NetworkInfo{}
context.ConfigDir = k8sutil.DataDir
context.Clientset = clientset
context.APIExtensionClientset = apiExtClientset
context.RookClientset = rookClientset

1.3 获得运行operator进程的pod

namespace为rook-ceph，podname以及镜像rook/ceph:master

// Using the current image version to deploy other rook pods
pod, err := k8sutil.GetRunningPod(clientset)
if err != nil {
	rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
}

rookImage, err := k8sutil.GetContainerImage(pod, containerName)
if err != nil {
	rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
}

1.4 创建operator实例

// New creates an operator instance
func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage, securityAccount string) *Operator {
	clusterController := cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper)

	schemes := []opkit.CustomResource{cluster.ClusterResource, pool.PoolResource, object.ObjectStoreResource, objectuser.ObjectStoreUserResource,
		file.FilesystemResource, attachment.VolumeResource}
	return &Operator{
		context:           context,
		clusterController: clusterController,
		resources:         schemes,
		rookImage:         rookImage,
		securityAccount:   securityAccount,
	}
}

2. agent Start函数

主要工作是创建daemonset的pod

rookAgent := agent.New(o.context.Clientset)

if err := rookAgent.Start(namespace, o.rookImage, o.securityAccount); err != nil {
	return fmt.Errorf("Error starting agent daemonset: %v", err)
}

2.1 createAgentDaemonSet函数

2.1.1 FLEXVOLUME_DIR_PATH变量默认为/usr/libexec/kubernetes/kubelet-plugins/volume/exec/

curl localhost:8080/api/v1/nodes/master-node/proxy/configz

{
    "kubeletconfig":{
        "syncFrequency":"1m0s",
        "fileCheckFrequency":"20s",
        "httpCheckFrequency":"20s",
        "address":"192.168.74.57",
        "port":10250,
        "readOnlyPort":10255,
        "tlsCertFile":"/etc/kubernetes/ssl/kubelet.crt",
        "tlsPrivateKeyFile":"/etc/kubernetes/ssl/kubelet.key",
        "authentication":{
            "x509":{

            },
            "webhook":{
                "enabled":false,
                "cacheTTL":"2m0s"
            },
            "anonymous":{
                "enabled":true
            }
        },
        "authorization":{
            "mode":"AlwaysAllow",
            "webhook":{
                "cacheAuthorizedTTL":"5m0s",
                "cacheUnauthorizedTTL":"30s"
            }
        },
        "registryPullQPS":5,
        "registryBurst":10,
        "eventRecordQPS":5,
        "eventBurst":10,
        "enableDebuggingHandlers":true,
        "healthzPort":10248,
        "healthzBindAddress":"127.0.0.1",
        "oomScoreAdj":-999,
        "clusterDomain":"cluster.local.",
        "clusterDNS":[
            "10.200.254.254"
        ],
        "streamingConnectionIdleTimeout":"4h0m0s",
        "nodeStatusUpdateFrequency":"10s",
        "nodeStatusReportFrequency":"1m0s",
        "nodeLeaseDurationSeconds":40,
        "imageMinimumGCAge":"2m0s",
        "imageGCHighThresholdPercent":85,
        "imageGCLowThresholdPercent":80,
        "volumeStatsAggPeriod":"1m0s",
        "cgroupsPerQOS":true,
        "cgroupDriver":"cgroupfs",
        "cpuManagerPolicy":"none",
        "cpuManagerReconcilePeriod":"10s",
        "runtimeRequestTimeout":"2m0s",
        "hairpinMode":"hairpin-veth",
        "maxPods":110,
        "podPidsLimit":-1,
        "resolvConf":"/etc/resolv.conf",
        "cpuCFSQuota":true,
        "cpuCFSQuotaPeriod":"100ms",
        "maxOpenFiles":1000000,
        "contentType":"application/vnd.kubernetes.protobuf",
        "kubeAPIQPS":5,
        "kubeAPIBurst":10,
        "serializeImagePulls":true,
        "evictionHard":{
            "imagefs.available":"15%",
            "memory.available":"100Mi",
            "nodefs.available":"10%",
            "nodefs.inodesFree":"5%"
        },
        "evictionPressureTransitionPeriod":"5m0s",
        "enableControllerAttachDetach":true,
        "makeIPTablesUtilChains":true,
        "iptablesMasqueradeBit":14,
        "iptablesDropBit":15,
        "failSwapOn":false,
        "containerLogMaxSize":"10Mi",
        "containerLogMaxFiles":5,
        "configMapAndSecretChangeDetectionStrategy":"Watch",
        "enforceNodeAllocatable":[
            "pods"
        ]
    }
}

flexvolumeDirPath, source := a.discoverFlexvolumeDir()
logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath)

2.1.2 agent mount security mode设置为Any

agentMountSecurityMode := os.Getenv(AgentMountSecurityModeEnv)
if agentMountSecurityMode == "" {
	logger.Infof("no agent mount security mode given, defaulting to '%s' mode", MountSecurityModeAny)
	agentMountSecurityMode = MountSecurityModeAny
}
if agentMountSecurityMode != MountSecurityModeAny && agentMountSecurityMode != MountSecurityModeRestricted {
	return fmt.Errorf("invalid agent mount security mode specified (given: %s)", agentMountSecurityMode)
}

2.1.3 创建daemonset名为rook-ceph-agent，启动参数为ceph agent，特权模式，挂盘设置，/flexmnt /dev /sys /lib/modules

简单粗暴直接构建daemon结构

2.1.4 调用client-go API创建daemonset，比较简单

3. discover Start函数

3.1 createDiscoverDaemonSet

这个比较简单，调用client-go API直接创建名为disove的daemonset

4. 开启CSI driver

如果operator开启了ROOK_CSI_ENABLE_CEPHFS或者ROOK_CSI_ENABLE_RBD则执行CSI流程

# CSI enablement
- name: ROOK_CSI_ENABLE_CEPHFS
  value: "true"
- name: ROOK_CSI_CEPHFS_IMAGE
  value: "quay.io/cephcsi/cephfsplugin:v1.0.0"
- name: ROOK_CSI_ENABLE_RBD
  value: "true"
- name: ROOK_CSI_RBD_IMAGE
  value: "quay.io/cephcsi/rbdplugin:v1.0.0"
- name: ROOK_CSI_REGISTRAR_IMAGE
  value: "quay.io/k8scsi/csi-node-driver-registrar:v1.0.2"
- name: ROOK_CSI_PROVISIONER_IMAGE
  value: "quay.io/k8scsi/csi-provisioner:v1.0.1"
- name: ROOK_CSI_SNAPSHOTTER_IMAGE
  value: "quay.io/k8scsi/csi-snapshotter:v1.0.1"
- name: ROOK_CSI_ATTACHER_IMAGE
  value: "quay.io/k8scsi/csi-attacher:v1.0.1"

调用StartCSIDrivers执行主要逻辑

if serverVersion.Major >= csi.KubeMinMajor && serverVersion.Minor >= csi.KubeMinMinor && csi.CSIEnabled() {
	logger.Infof("Ceph CSI driver is enabled, validate csi param")
	if err = csi.ValidateCSIParam(); err != nil {
		logger.Warningf("invalid csi params: %v", err)
		if csi.ExitOnError {
			return err
		}
	} else {
		csi.SetCSINamespace(namespace)
		if err = csi.StartCSIDrivers(namespace, o.context.Clientset); err != nil {
			logger.Warningf("failed to start Ceph csi drivers: %v", err)
			if csi.ExitOnError {
				return err
			}
		} else {
			logger.Infof("successfully started Ceph csi drivers")
		}
	}
}

5. StartCSIDrivers函数

5.1 rbdplugin模板生成daemonset配置

默认路径 /etc/ceph-csi/rbd/csi-rbdplugin.yaml，将变量的地方替换未CSIParam全局变量中的数据

启动包括driver-registart，csi-rbdplugin两个容器共用一个pod，使用unix:///csi/csi.sock通信

kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: csi-rbdplugin
  namespace: {{ .Namespace }}  
spec:
  selector:
    matchLabels:
      app: csi-rbdplugin
  template:
    metadata:
      labels:
        app: csi-rbdplugin
    spec:
      serviceAccount: rook-csi-rbd-plugin-sa
      hostNetwork: true
      hostPID: true      
      # to use e.g. Rook orchestrated cluster, and mons' FQDN is
      # resolved through k8s service, set dns policy to cluster first
      dnsPolicy: ClusterFirstWithHostNet      
      containers:
        - name: driver-registrar
          image: {{ .RegistrarImage }}
          args:
            - "--v=5"
            - "--csi-address=/csi/csi.sock"
            - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock"
          lifecycle:
            preStop:
              exec:
                  command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"]          
          env:
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
        - name: csi-rbdplugin
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
          image: {{ .RBDPluginImage }}
          args :
            - "--nodeid=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--v=5"
            - "--drivername=rbd.csi.ceph.com"
            - "--containerized=true"
            - "--metadatastorage=k8s_configmap"
          env:
            - name: HOST_ROOTFS
              value: "/rootfs" 
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
          imagePullPolicy: "IfNotPresent"

func templateToDaemonSet(name, templatePath string) (*apps.DaemonSet, error) {
	var ds apps.DaemonSet
	t, err := loadTemplate(name, templatePath)
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal([]byte(t), &ds)
	if err != nil {
		return nil, err
	}
	return &ds, nil
}

5.2 rbd-provisioner模板生成statefulel配置

默认路径 /etc/ceph-csi/rbd/csi-rbdplugin-provisioner.yaml

启动了statefulset的pod，包括容器provisioner，attacher，snapshotter，rbdplugin四个容器，使用unix:///csi/csi-provisioner.sock通信

kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: csi-rbdplugin-provisioner
  namespace: {{ .Namespace }}
spec:
  serviceName: "csi-rbdplugin-provisioner"
  replicas: 1
  selector:
    matchLabels:
     app: csi-rbdplugin-provisioner
  template:
    metadata:
      labels:
        app: csi-rbdplugin-provisioner
    spec:
      serviceAccount: rook-csi-rbd-provisioner-sa
      containers:
        - name: csi-provisioner
          image: {{ .ProvisionerImage }}
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=5"
          env:
            - name: ADDRESS
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-rbdplugin-attacher
          image: {{ .AttacherImage }}
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-snapshotter
          image:  {{ .SnapshotterImage }}
          args:
            - "--csi-address=$(ADDRESS)"
            - "--connection-timeout=15s"
            - "--v=5"
          env:
            - name: ADDRESS
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: Always
          securityContext:
            privileged: true
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
        - name: csi-rbdplugin
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
          image: {{ .RBDPluginImage }}
          args :
            - "--nodeid=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--v=5"
            - "--drivername=rbd.csi.ceph.com"
            - "--containerized=true"
            - "--metadatastorage=k8s_configmap"
          env:
            - name: HOST_ROOTFS
              value: "/rootfs"
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: CSI_ENDPOINT
              value: unix:///csi/csi-provisioner.sock
          imagePullPolicy: "IfNotPresent"

func templateToStatefulSet(name, templatePath string) (*apps.StatefulSet, error) {
	var ss apps.StatefulSet
	t, err := loadTemplate(name, templatePath)
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal([]byte(t), &ss)
	if err != nil {
		return nil, err
	}
	return &ss, nil
}

如果开启cephfs也同样处理

5.3 创建rbdPlugin的daemonset，创建rdbprovisioner的statefuleSet

if rbdPlugin != nil {
	err = k8sutil.CreateDaemonSet("csi rbd plugin", namespace, clientset, rbdPlugin)
	if err != nil {
		return fmt.Errorf("failed to start rbdplugin daemonset: %v\n%v", err, rbdPlugin)
	}
}
if rbdProvisioner != nil {
	_, err = k8sutil.CreateStatefulSet("csi rbd provisioner", namespace, "csi-rbdplugin-provisioner", clientset, rbdProvisioner)
	if err != nil {
		return fmt.Errorf("failed to start rbd provisioner statefulset: %v\n%v", err, rbdProvisioner)
	}

}

5.4 同样创建cephfs的daemon与provisioner

if cephfsPlugin != nil {
	err = k8sutil.CreateDaemonSet("csi cephfs plugin", namespace, clientset, cephfsPlugin)
	if err != nil {
		return fmt.Errorf("failed to start cephfs plugin daemonset: %v\n%v", err, cephfsPlugin)
	}
}
if cephfsProvisioner != nil {
	_, err = k8sutil.CreateStatefulSet("csi cephfs provisioner", namespace, "csi-cephfsplugin-provisioner", clientset, cephfsProvisioner)
	if err != nil {
		return fmt.Errorf("failed to start cephfs provisioner statefulset: %v\n%v", err, cephfsProvisioner)
	}

}

provisionerName       = "ceph.rook.io/block"
provisionerNameLegacy = "rook.io/block"

6. NewProvisionController

RookVolumeProvisioner实现了Provisioner接口，包括方法Provision与Delete

与controller manager形式一样，创建controller

// NewProvisionController creates a new provision controller using
// the given configuration parameters and with private (non-shared) informers.
func NewProvisionController(
	client kubernetes.Interface,
	provisionerName string,
	provisioner Provisioner,
	kubeVersion string,
	options ...func(*ProvisionController) error,
) *ProvisionController {
	id, err := os.Hostname()
	if err != nil {
		glog.Fatalf("Error getting hostname: %v", err)
	}

两个队列,claims与volumes，也就是pv pvc资源队列

	controller.claimQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "claims")
	controller.volumeQueue = workqueue.NewNamedRateLimitingQueue(rateLimiter, "volumes")

PVC回调函数Add Update Delete

	// ----------------------
	// PersistentVolumeClaims

	claimHandler := cache.ResourceEventHandlerFuncs{
		AddFunc:    func(obj interface{}) { controller.enqueueClaim(obj) },
		UpdateFunc: func(oldObj, newObj interface{}) { controller.enqueueClaim(newObj) },
		DeleteFunc: func(obj interface{}) {
			// NOOP. The claim is either in claimsInProgress and in the queue, so it will be processed as usual
			// or it's not in claimsInProgress and then we don't care
		},
	}

PV回调函数

	// -----------------
	// PersistentVolumes

	volumeHandler := cache.ResourceEventHandlerFuncs{
		AddFunc:    func(obj interface{}) { controller.enqueueVolume(obj) },
		UpdateFunc: func(oldObj, newObj interface{}) { controller.enqueueVolume(newObj) },
		DeleteFunc: func(obj interface{}) { controller.forgetVolume(obj) },
	}

storageclass资源

	// --------------
	// StorageClasses

	// no resource event handler needed for StorageClasses
	if controller.classInformer == nil {
		if controller.kubeVersion.AtLeast(utilversion.MustParseSemantic("v1.6.0")) {
			controller.classInformer = informer.Storage().V1().StorageClasses().Informer()
		} else {
			controller.classInformer = informer.Storage().V1beta1().StorageClasses().Informer()
		}
	}
	controller.classes = controller.classInformer.GetStore()

剩下的流程与其他controller manger中处理一样，可以参考https://blog.csdn.net/zhonglinzhang/article/details/89915182文章

7. 开启watch rook cluster功能

也就是CephCluster资源，这篇文章分析

https://blog.csdn.net/zhonglinzhang/article/details/89845140

// watch for changes to the rook clusters
o.clusterController.StartWatch(namespaceToWatch, stopChan)

总结

operator启动创建agent与discover的daemonset的服务

如果开启ceph csi则创建CSI插件，statefulset与daemon

创建controller，使用informer机制关注pv pvc storageclass

开启watch CephCluster资源进行处理

张忠琳

关注

2
点赞
踩
0

收藏

觉得还不错? 一键收藏
1
评论
【kubernetes/k8s源码分析】 rook operator启动源码分析

https://github.com/rook/rook operator：作用启动监控存储集群，包括mon，osd，mgr等服务，可以支持 block 块存储，file system 文件存储，object 对象存储 (S3/Swift)。启动包括ceph agent 与 discover服务，agent 包含Flexvolume plugin，包括attaching n...
复制链接

扫一扫

专栏目录