【kubernetes/k8s源码分析】 kubernetes csi node driver registrar 源码分析

Git Repository: https://github.com/kubernetes-csi/node-driver-registrar

Status: GA/Stable

Latest stable release	Branch	Min CSI Version	Max CSI Version	Container Image	Min k8s Version	Max k8s version
node-driver-registrar v1.1.0	release-1.1	v1.0.0	-	quay.io/k8scsi/csi-node-driver-registrar:v1.1.0	v1.13	-
node-driver-registrar v1.0.2	release-1.0	v1.0.0	-	quay.io/k8scsi/csi-node-driver-registrar:v1.0.2	v1.13	-
driver-registrar v0.4.2	release-0.4	v0.3.0	v0.3.0	quay.io/k8scsi/driver-registrar:v0.4.2	v1.10	-

 

概述

      负责请求 Identity Service 来获取插件信息并且注册到 kubelet，初始化的时候通过 csi-sock 的 RPC 获取 driver name 和 node id。主要功能给 node 打上 annotations，device-driver-registar 主要是注册 Node annotation

metadata:
  annotations:
    csi.volume.kubernetes.io/nodeid: '{"cephfs.csi.ceph.com":"node1","rbd.csi.ceph.com":"node1"}'
    node.alpha.kubernetes.io/ttl: "0"
    volumes.kubernetes.io/controller-managed-attach-detach: "true"

 

Registration socket:

• Registers the driver with kubelet.

• Created by the node-driver-registrar.

• Exposed on a Kubernetes node via hostpath in the Kubelet plugin registry. (typically /var/lib/kubelet/plugins_registry/<drivername.example.com>-reg.sock). The hostpath volume must be mounted at /registration

CSI driver socket:

• Used by kubelet to interact with the CSI driver.
• Created by the CSI driver.
• Exposed on a Kubernetes node via hostpath somewhere other than the Kubelet. plugin registry (typically /var/lib/kubelet/plugins/<drivername.example.com>/csi.sock).
• This is the socket referenced by the --csi-address and --kubelet-registration-path arguments.

 

    ① node-driver 调用 CSI-plugin 插件的 GetPluginInfo 获得 driver 名字

    ② node-driver 创建 GRPC-Server，socket 地址在宿主机 /var/lib/kubelet/plugins_registry/${driver-name}-reg.sock，容器里为 /registration/${driver-name}-reg.sock

    ③ kubelet 与 node-driver  建立 GRPC 连接，调用 GetInfo 获得插件信息，包括 plugin 名字，GRPC-Server socket 地址，这个是 Watcher 目录，为 socket 

    ④ kubelet 向 CSI-Plugin 发送 NodeGetInfo GRPC 请求，获得节点信息，将 CSI-Plugin 加入插件列表

    ⑤ kubelet 更新 node 资源，主要是更新注解 Annotations，创建 CSINode 资源

    ⑥ kubelet 调用 NotifyRegistrationStatus 通知 node-driver 是否成功

 

启动命令

     si-address：CSI driver socket，node-driver-registrar将连到这socket上

     kubelet-registration-path：在kubelet节点的CSI driver socket路径

     node-driver-registrar --v=5

--csi-address=/csi/csi.sock

--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock 

 

 

1. main函数

    1.1 与CSI driver的socket建立GRPC

klog.V(1).Infof("Attempting to open a gRPC connection with: %q", *csiAddress)
csiConn, err := connection.Connect(*csiAddress)
if err != nil {
	klog.Errorf("error connecting to CSI driver: %v", err)
	os.Exit(1)
}

     1.2 GetDriverName函数返回CSI driver

       发送 GetPluginInfoRequest GRPC 请求 GetPluginInfo，例如 rbd.csi.ceph.com

// GetDriverName returns name of CSI driver.
func GetDriverName(ctx context.Context, conn *grpc.ClientConn) (string, error) {
	client := csi.NewIdentityClient(conn)

	req := csi.GetPluginInfoRequest{}
	rsp, err := client.GetPluginInfo(ctx, &req)
	if err != nil {
		return "", err
	}
	name := rsp.GetName()
	if name == "" {
		return "", fmt.Errorf("driver name is empty")
	}
	return name, nil
}

 

2. nodeRegister函数

    监听 /registration/rbd.csi.ceph.com-reg.sock

registrar := newRegistrationServer(csiDriverName, *kubeletRegistrationPath, supportedVersions)
socketPath := fmt.Sprintf("/registration/%s-reg.sock", csiDriverName)
fi, err := os.Stat(socketPath)
if err == nil && (fi.Mode()&os.ModeSocket) != 0 {
	// Remove any socket, stale or not, but fall through for other files
	if err := os.Remove(socketPath); err != nil {
		klog.Errorf("failed to remove stale socket %s with error: %+v", socketPath, err)
		os.Exit(1)
	}
}
if err != nil && !os.IsNotExist(err) {
	klog.Errorf("failed to stat the socket %s with error: %+v", socketPath, err)
	os.Exit(1)
}
// Default to only user accessible socket, caller can open up later if desired
oldmask := unix.Umask(0077)

klog.Infof("Starting Registration Server at: %s\n", socketPath)
lis, err := net.Listen("unix", socketPath)
if err != nil {
	klog.Errorf("failed to listen on socket: %s with error: %+v", socketPath, err)
	os.Exit(1)
}

    2.1 Registers kubelet plugin watcher api

       建立GRPC server服务

grpcServer := grpc.NewServer()
// Registers kubelet plugin watcher api.
registerapi.RegisterRegistrationServer(grpcServer, registrar)

// Starts service
if err := grpcServer.Serve(lis); err != nil {
	klog.Errorf("Registration Server stopped serving: %v", err)
	os.Exit(1)
}

 

3. newRegistrationServer实例化registrationServer

// NewregistrationServer returns an initialized registrationServer instance
func newRegistrationServer(driverName string, endpoint string, versions []string) registerapi.RegistrationServer {
	return &registrationServer{
		driverName: driverName,
		endpoint:   endpoint,
		version:    versions,
	}
}

    实现了两个接口，在 kubelet 发送 GRPC 请求时使用

// GetInfo is the RPC invoked by plugin watcher
func (e registrationServer) GetInfo(ctx context.Context, req *registerapi.InfoRequest) (*registerapi.PluginInfo, error) {
	klog.Infof("Received GetInfo call: %+v", req)
	return &registerapi.PluginInfo{
		Type:              registerapi.CSIPlugin,
		Name:              e.driverName,
		Endpoint:          e.endpoint,
		SupportedVersions: e.version,
	}, nil
}

func (e registrationServer) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) {
	klog.Infof("Received NotifyRegistrationStatus call: %+v", status)
	if !status.PluginRegistered {
		klog.Errorf("Registration process failed with error: %+v, restarting registration container.", status.Error)
		os.Exit(1)
	}

	return &registerapi.RegistrationStatusResponse{}, nil
}

 

总结：

    node driver registrar 作为 GRPC 客户端，与 CSI socket 建立 GRPC 连接，发送 GetPluginInfoRequest 请求获得 CSI driver 名字，与 CSI plugin 进行交互

    在 /registration/%s-reg.sock 建立 GRPC server 监听，这个与 kubelet 进行交互

    

 

Dockerfile

FROM gcr.io/distroless/static:latest
LABEL maintainers="Kubernetes Authors"
LABEL description="CSI Node driver registrar"

COPY ./bin/csi-node-driver-registrar csi-node-driver-registrar
ENTRYPOINT ["/csi-node-driver-registrar"]

 

参考：

     https://github.com/kubernetes-csi/node-driver-registrar