kubernetes 源码分析之kubeadm（三）

上一篇介绍了kubeadm的init这篇主要讲解join，它们我们用kubeadm最常用的两个组件。
和init的一样，首先还是配置解析参数，创建join结构体

func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, skipPreFlight bool) (*Join, error) {
    fmt.Println("[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.")

    if cfgPath != "" {
        b, err := ioutil.ReadFile(cfgPath)
        if err != nil {
            return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
        }
        if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
            return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
        }
    }

    if !skipPreFlight {
        fmt.Println("[preflight] Running pre-flight checks")

        // First, check if we're root separately from the other preflight checks and fail fast
        if err := preflight.RunRootCheckOnly(); err != nil {
            return nil, err
        }

        // Then continue with the others...
        if err := preflight.RunJoinNodeChecks(cfg); err != nil {
            return nil, err
        }
    } else {
        fmt.Println("[preflight] Skipping pre-flight checks")
    }

    // Try to start the kubelet service in case it's inactive
    preflight.TryStartKubelet()

    return &Join{cfg: cfg}, nil
}

skipPreFlight这类的也是和init一样，只不过检查的内容不一样

checks := []Checker{
        SystemVerificationCheck{},
        IsRootCheck{},
        HostnameCheck{},
        ServiceCheck{Service: "kubelet", CheckIfActive: false},
        ServiceCheck{Service: "docker", CheckIfActive: true},
        PortOpenCheck{port: 10250},
        DirAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")},
        DirAvailableCheck{Path: "/var/lib/kubelet"},
        FileAvailableCheck{Path: cfg.CACertPath},
        FileAvailableCheck{Path: filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)},
        FileContentCheck{Path: bridgenf, Content: []byte{'1'}},
        InPathCheck{executable: "ip", mandatory: true},
        InPathCheck{executable: "iptables", mandatory: true},
        InPathCheck{executable: "mount", mandatory: true},
        InPathCheck{executable: "nsenter", mandatory: true},
        InPathCheck{executable: "ebtables", mandatory: false},
        InPathCheck{executable: "ethtool", mandatory: false},
        InPathCheck{executable: "socat", mandatory: false},
        InPathCheck{executable: "tc", mandatory: false},
        InPathCheck{executable: "touch", mandatory: false},
    }

然后是执行命令：

func (j *Join) Run(out io.Writer) error {
    cfg, err := discovery.For(j.cfg)
    if err != nil {
        return err
    }

    hostname, err := os.Hostname()
    if err != nil {
        return err
    }
    client, err := kubeconfigutil.KubeConfigToClientSet(cfg)
    if err != nil {
        return err
    }
    if err := kubenode.ValidateAPIServer(client); err != nil {
        return err
    }
    if err := kubenode.PerformTLSBootstrap(cfg, hostname); err != nil {
        return err
    }

    kubeconfigFile := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
    if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil {
        return err
    }

    // Write the ca certificate to disk so kubelet can use it for authentication
    cluster := cfg.Contexts[cfg.CurrentContext].Cluster
    err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData)
    if err != nil {
        return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
    }

    fmt.Fprintf(out, joinDoneMsgf)
    return nil
}

它不init要简单，首先是获取主机名，创建k8s api调用的client，然后通过客户端验证版本，执行节点的签名请求kubelet-csr，然后生成kubelet的kubeconfig，这样kubelet就可以加入到集群里面了。这里有个点需要交代清楚。
第一是在初始化cfg的时候是把v1alpha1转成内部版本。并且在此之前还有Defualt过程，

func SetDefaults_NodeConfiguration(obj *NodeConfiguration) {
    if obj.CACertPath == "" {
        obj.CACertPath = DefaultCACertPath
    }
    if len(obj.TLSBootstrapToken) == 0 {
        obj.TLSBootstrapToken = obj.Token
    }
    if len(obj.DiscoveryToken) == 0 && len(obj.DiscoveryFile) == 0 {
        obj.DiscoveryToken = obj.Token
    }
    // Make sure file URLs become paths
    if len(obj.DiscoveryFile) != 0 {
        u, err := url.Parse(obj.DiscoveryFile)
        if err == nil && u.Scheme == "file" {
            obj.DiscoveryFile = u.Path
        }
    }
}

这个Default()函数中最重要的是obj.TLSBootstrapToken = obj.Token，这个在后面会遇到，请大家记住这个赋值。
第二个问题是获取获取初始化cluster-info信息cmd/kubeadm/app/discovery/token/token.go。

var clusterinfo *v1.ConfigMap
        wait.PollInfinite(constants.DiscoveryRetryInterval, func() (bool, error) {
            var err error
            clusterinfo, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
            if err != nil {
                fmt.Printf("[discovery] Failed to request cluster info, will try again: [%s]\n", err)
                return false, nil
            }
            return true, nil
        })

掉用k8s api获取cluster-info信息。从而获取kubeconfig。
第三个问题是签名认证cmd/kubeadm/app/node/csr.go：

func PerformTLSBootstrap(cfg *clientcmdapi.Config, hostName string) error {
    client, err := kubeconfigutil.KubeConfigToClientSet(cfg)
    if err != nil {
        return err
    }

    fmt.Println("[csr] Created API client to obtain unique certificate for this node, generating keys and certificate signing request")

    key, err := certutil.MakeEllipticPrivateKeyPEM()
    if err != nil {
        return fmt.Errorf("failed to generate private key [%v]", err)
    }

    cert, err := csr.RequestNodeCertificate(client.CertificatesV1beta1().CertificateSigningRequests(), key, types.NodeName(hostName))
    if err != nil {
        return fmt.Errorf("failed to request signed certificate from the API server [%v]", err)
    }
    fmt.Println("[csr] Received signed certificate from the API server, generating KubeConfig...")

    cfg.AuthInfos[CSRContextAndUser] = &clientcmdapi.AuthInfo{
        ClientKeyData:         key,
        ClientCertificateData: cert,
    }
    cfg.Contexts[CSRContextAndUser] = &clientcmdapi.Context{
        AuthInfo: CSRContextAndUser,
        Cluster:  cfg.Contexts[cfg.CurrentContext].Cluster,
    }
    cfg.CurrentContext = CSRContextAndUser
    return nil
}

先生存key，然后执行RequestNodeCertificate，这个方法先创建csr，然后发送给api，如果签名成功就返回签名过后的证书。这为后续生存kubeconfig里面的client-certificate-data准备的参数

func RequestNodeCertificate(client certificatesclient.CertificateSigningRequestInterface, privateKeyData []byte, nodeName types.NodeName) (certData []byte, err error) {
    subject := &pkix.Name{
        Organization: []string{"system:nodes"},
        CommonName:   fmt.Sprintf("system:node:%s", nodeName),
    }

    privateKey, err := certutil.ParsePrivateKeyPEM(privateKeyData)
    if err != nil {
        return nil, fmt.Errorf("invalid private key for certificate request: %v", err)
    }
    csrData, err := certutil.MakeCSR(privateKey, subject, nil, nil)
    if err != nil {
        return nil, fmt.Errorf("unable to generate certificate request: %v", err)
    }
    return RequestCertificate(client, csrData, []certificates.KeyUsage{
        certificates.UsageDigitalSignature,
        certificates.UsageKeyEncipherment,
        certificates.UsageClientAuth,
    })
}

那么这个join就完成了，当然kubeadm除了join还有一些别的命令如reset，这个命令比较简单粗暴简单我大概说一下：

func (r *Reset) Run(out io.Writer) error {

    // Try to stop the kubelet service
    initSystem, err := initsystem.GetInitSystem()
    if err != nil {
        fmt.Println("[reset] WARNING: The kubelet service couldn't be stopped by kubeadm because no supported init system was detected.")
        fmt.Println("[reset] WARNING: Please ensure kubelet is stopped manually.")
    } else {
        fmt.Println("[reset] Stopping the kubelet service")
        if err := initSystem.ServiceStop("kubelet"); err != nil {
            fmt.Printf("[reset] WARNING: The kubelet service couldn't be stopped by kubeadm: [%v]\n", err)
            fmt.Println("[reset] WARNING: Please ensure kubelet is stopped manually.")
        }
    }

    // Try to unmount mounted directories under /var/lib/kubelet in order to be able to remove the /var/lib/kubelet directory later
    fmt.Printf("[reset] Unmounting mounted directories in %q\n", "/var/lib/kubelet")
    umountDirsCmd := "cat /proc/mounts | awk '{print $2}' | grep '/var/lib/kubelet' | xargs -r umount"
    umountOutputBytes, err := exec.Command("sh", "-c", umountDirsCmd).Output()
    if err != nil {
        fmt.Printf("[reset] Failed to unmount mounted directories in /var/lib/kubelet: %s\n", string(umountOutputBytes))
    }

    dockerCheck := preflight.ServiceCheck{Service: "docker", CheckIfActive: true}
    if warnings, errors := dockerCheck.Check(); len(warnings) == 0 && len(errors) == 0 {
        fmt.Println("[reset] Removing kubernetes-managed containers")
        if err := exec.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil {
            fmt.Println("[reset] Failed to stop the running containers")
        }
    } else {
        fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers")
    }

    dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim"}

    // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user
    // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd
    etcdManifestPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests/etcd.yaml")
    if _, err := os.Stat(etcdManifestPath); err == nil {
        dirsToClean = append(dirsToClean, "/var/lib/etcd")
    } else {
        fmt.Printf("[reset] No etcd manifest found in %q, assuming external etcd.\n", etcdManifestPath)
    }

    // Then clean contents from the stateful kubelet, etcd and cni directories
    fmt.Printf("[reset] Deleting contents of stateful directories: %v\n", dirsToClean)
    for _, dir := range dirsToClean {
        cleanDir(dir)
    }

    // Remove contents from the config and pki directories
    resetConfigDir(kubeadmapi.GlobalEnvParams.KubernetesDir, r.certsDir)

    return nil
}

就是把init和join的生成的东西都给删了。详细删目主要是dirsToClean := []string{“/var/lib/kubelet”, “/etc/cni/net.d”, “/var/lib/dockershim”}等，删除的方法

func cleanDir(filePath string) error {
    // If the directory doesn't even exist there's nothing to do, and we do
    // not consider this an error
    if _, err := os.Stat(filePath); os.IsNotExist(err) {
        return nil
    }

    d, err := os.Open(filePath)
    if err != nil {
        return err
    }
    defer d.Close()
    names, err := d.Readdirnames(-1)
    if err != nil {
        return err
    }
    for _, name := range names {
        err = os.RemoveAll(filepath.Join(filePath, name))
        if err != nil {
            return err
        }
    }
    return nil
}

删除之前创建的目录和文件。其它命令就没啥了。简单总结一下，kubeadm就是通过kubelet自举master以及daemonset组合启动k8s相关的组件。