kubernetes容器启动详解

如果大家对kubernetes组件以及架构相关分析，可以看我之前的源码阅读，今天只从一个函数分析容器的启动过程，这个函数就是SyncPod，这个是创建kubelet里面最核心的一个函数了。这个方法分为五步：

//  1. Compute sandbox and container changes.
//  2. Kill pod sandbox if necessary.
//  3. Kill any containers that should not be running.
//  4. Create sandbox if necessary.
//  5. Create init containers.
//  6. Create normal containers.

下面就逐一讲解：

1.计算容器的变化

这部分代码主要完成容器状态的变化统计，就是确定哪些容器要创建，哪些容器要删除。

    // Step 1: Compute sandbox and container changes.
    podContainerChanges := m.computePodContainerChanges(pod, podStatus)
    glog.V(3).Infof("computePodContainerChanges got %+v for pod %q", podContainerChanges, format.Pod(pod))
    if podContainerChanges.CreateSandbox {
        ref, err := ref.GetReference(api.Scheme, pod)
        if err != nil {
            glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err)
        }
        if podContainerChanges.SandboxID != "" {
            m.recorder.Eventf(ref, v1.EventTypeNormal, "SandboxChanged", "Pod sandbox changed, it will be killed and re-created.")
        } else {
            glog.V(4).Infof("SyncPod received new pod %q, will create a new sandbox for it", format.Pod(pod))
        }
    }

具体实现看m.computePodContainerChanges，如下：

func (m *kubeGenericRuntimeManager) computePodContainerChanges(pod *v1.Pod, podStatus *kubecontainer.PodStatus) podContainerSpecChanges {
    glog.V(5).Infof("Syncing Pod %q: %+v", format.Pod(pod), pod)

    sandboxChanged, attempt, sandboxID := m.podSandboxChanged(pod, podStatus)
    changes := podContainerSpecChanges{
        CreateSandbox:        sandboxChanged,
        SandboxID:            sandboxID,
        Attempt:              attempt,
        ContainersToStart:    make(map[int]string),
        ContainersToKeep:     make(map[kubecontainer.ContainerID]int),
        InitContainersToKeep: make(map[kubecontainer.ContainerID]int),
        ContainersToKill:     make(map[kubecontainer.ContainerID]containerToKillInfo),
    }

    // check the status of init containers.
    initFailed := false
    // always reset the init containers if the sandbox is changed.
    if !sandboxChanged {
        // Keep all successfully completed containers. If there are failing containers,
        // only keep the first failing one.
        initFailed = checkAndKeepInitContainers(pod, podStatus, changes.InitContainersToKeep)
    }
    changes.InitFailed = initFailed

    // check the status of containers.
    for index, container := range pod.Spec.Containers {
        containerStatus := podStatus.FindContainerStatusByName(container.Name)
        if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning {
            if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) {
                message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container)
                glog.Info(message)
                changes.ContainersToStart[index] = message
            }
            continue
        }
        if sandboxChanged {
            if pod.Spec.RestartPolicy != v1.RestartPolicyNever {
                message := fmt.Sprintf("Container %+v's pod sandbox is dead, the container will be recreated.", container)
                glog.Info(message)
                changes.ContainersToStart[index] = message
            }
            continue
        }

        if initFailed {
            // Initialization failed and Container exists.
            // If we have an initialization failure everything will be killed anyway.
            // If RestartPolicy is Always or OnFailure we restart containers that were running before.
            if pod.Spec.RestartPolicy != v1.RestartPolicyNever {
                message := fmt.Sprintf("Failed to initialize pod. %q will be restarted.", container.Name)
                glog.V(1).Info(message)
                changes.ContainersToStart[index] = message
            }
            continue
        }

        expectedHash := kubecontainer.HashContainer(&container)
        containerChanged := containerStatus.Hash != expectedHash
        if containerChanged {
            message := fmt.Sprintf("Pod %q container %q hash changed (%d vs %d), it will be killed and re-created.",
                pod.Name, container.Name, containerStatus.Hash, expectedHash)
            glog.Info(message)
            changes.ContainersToStart[index] = message
            continue
        }

        liveness, found := m.livenessManager.Get(containerStatus.ID)
        if !found || liveness == proberesults.Success {
            changes.ContainersToKeep[containerStatus.ID] = index
            continue
        }
        if pod.Spec.RestartPolicy != v1.RestartPolicyNever {
            message := fmt.Sprintf("pod %q container %q is unhealthy, it will be killed and re-created.", format.Pod(pod), container.Name)
            glog.Info(message)
            changes.ContainersToStart[index] = message
        }
    }

    // Don't keep init containers if they are the only containers to keep.
    if !sandboxChanged && len(changes.ContainersToStart) == 0 && len(changes.ContainersToKeep) == 0 {
        changes.InitContainersToKeep = make(map[kubecontainer.ContainerID]int)
    }

    // compute containers to be killed
    runningContainerStatuses := podStatus.GetRunningContainerStatuses()
    for _, containerStatus := range runningContainerStatuses {
        _, keep := changes.ContainersToKeep[containerStatus.ID]
        _, keepInit := changes.InitContainersToKeep[containerStatus.ID]
        if !keep && !keepInit {
            var podContainer *v1.Container
            var killMessage string
            for i, c := range pod.Spec.Containers {
                if c.Name == containerStatus.Name {
                    podContainer = &pod.Spec.Containers[i]
                    killMessage = changes.ContainersToStart[i]
                    break
                }
            }

            changes.ContainersToKill[containerStatus.ID] = containerToKillInfo{
                name:      containerStatus.Name,
                container: podContainer,
                message:   killMessage,
            }
        }
    }

    return changes
}

计算出哪些容器要启动放到ContainersToStart这个管道里面，哪些容器要删除放到ContainersToKill这个管道里面。

2.删除sandbox变化的pod

这个主要是当沙箱变化的时候，需要重建pod，譬如切换了pause镜像，就会触发个操作

    // Step 2: Kill the pod if the sandbox has changed.
    if podContainerChanges.CreateSandbox || (len(podContainerChanges.ContainersToKeep) == 0 && len(podContainerChanges.ContainersToStart) == 0) {
        if len(podContainerChanges.ContainersToKeep) == 0 && len(podContainerChanges.ContainersToStart) == 0 {
            glog.V(4).Infof("Stopping PodSandbox for %q because all other containers are dead.", format.Pod(pod))
        } else {
            glog.V(4).Infof("Stopping PodSandbox for %q, will start new one", format.Pod(pod))
        }

        killResult := m.killPodWithSyncResult(pod, kubecontainer.ConvertPodStatusToRunningPod(m.runtimeName, podStatus), nil)
        result.AddPodSyncResult(killResult)
        if killResult.Error() != nil {
            glog.Errorf("killPodWithSyncResult failed: %v", killResult.Error())
            return
        }
    } 

3. 删除不需要运行的容器

当kubernetes执行删除pod的操作时候，有些pods是没必要存在的。

// Step 3: kill any running containers in this pod which are not to keep.
        for containerID, containerInfo := range podContainerChanges.ContainersToKill {
            glog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerInfo.name, containerID, format.Pod(pod))
            killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, containerInfo.name)
            result.AddSyncResult(killContainerResult)
            if err := m.killContainer(pod, containerID, containerInfo.name, containerInfo.message, nil); err != nil {
                killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error())
                glog.Errorf("killContainer %q(id=%q) for pod %q failed: %v", containerInfo.name, containerID, format.Pod(pod), err)
                return
            }
        }

下面是调用具体的容器运行时，去删除

func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubecontainer.ContainerID, containerName string, reason string, gracePeriodOverride *int64) error {
    var containerSpec *v1.Container
    if pod != nil {
        containerSpec = kubecontainer.GetContainerSpec(pod, containerName)
    } else {
        // Restore necessary information if one of the specs is nil.
        restoredPod, restoredContainer, err := m.restoreSpecsFromContainerLabels(containerID)
        if err != nil {
            return err
        }
        pod, containerSpec = restoredPod, restoredContainer
    }
    // From this point , pod and container must be non-nil.
    gracePeriod := int64(minimumGracePeriodInSeconds)
    switch {
    case pod.DeletionGracePeriodSeconds != nil:
        gracePeriod = *pod.DeletionGracePeriodSeconds
    case pod.Spec.TerminationGracePeriodSeconds != nil:
        gracePeriod = *pod.Spec.TerminationGracePeriodSeconds
    }

    glog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod)

    // Run the pre-stop lifecycle hooks if applicable.
    if containerSpec.Lifecycle != nil && containerSpec.Lifecycle.PreStop != nil {
        gracePeriod = gracePeriod - m.executePreStopHook(pod, containerID, containerSpec, gracePeriod)
    }
    // always give containers a minimal shutdown window to avoid unnecessary SIGKILLs
    if gracePeriod < minimumGracePeriodInSeconds {
        gracePeriod = minimumGracePeriodInSeconds
    }
    if gracePeriodOverride != nil {
        gracePeriod = *gracePeriodOverride
        glog.V(3).Infof("Killing container %q, but using %d second grace period override", containerID, gracePeriod)
    }

    err := m.runtimeService.StopContainer(containerID.ID, gracePeriod)
    if err != nil {
        glog.Errorf("Container %q termination failed with gracePeriod %d: %v", containerID.String(), gracePeriod, err)
    } else {
        glog.V(3).Infof("Container %q exited normally", containerID.String())
    }

    message := fmt.Sprintf("Killing container with id %s", containerID.String())
    if reason != "" {
        message = fmt.Sprint(message, ":", reason)
    }
    m.generateContainerEvent(containerID, v1.EventTypeNormal, events.KillingContainer, message)
    m.containerRefManager.ClearRef(containerID)

    return err

m.runtimeService.StopContainer带着优雅关闭时间的停止container，每种容器都是实现StopContainer这个接口，这个在kubelet启动的时候初始化的，之前代码已经分析过了，不再赘述。

4.创建沙箱

其实kubelet之所以引入沙箱，是想建立一个容器标准，这里可以简单理解成那个pause容器。所有的网络都是挂在这个基础容器里面。

    // Step 4: Create a sandbox for the pod if necessary.
    podSandboxID := podContainerChanges.SandboxID
    if podContainerChanges.CreateSandbox && len(podContainerChanges.ContainersToStart) > 0 {
        var msg string
        var err error

        glog.V(4).Infof("Creating sandbox for pod %q", format.Pod(pod))
        createSandboxResult := kubecontainer.NewSyncResult(kubecontainer.CreatePodSandbox, format.Pod(pod))
        result.AddSyncResult(createSandboxResult)
        podSandboxID, msg, err = m.createPodSandbox(pod, podContainerChanges.Attempt)
        if err != nil {
            createSandboxResult.Fail(kubecontainer.ErrCreatePodSandbox, msg)
            glog.Errorf("createPodSandbox for pod %q failed: %v", format.Pod(pod), err)
            return
        }
        glog.V(4).Infof("Created PodSandbox %q for pod %q", podSandboxID, format.Pod(pod))

        podSandboxStatus, err := m.runtimeService.PodSandboxStatus(podSandboxID)
        if err != nil {
            glog.Errorf("Failed to get pod sandbox status: %v; Skipping pod %q", err, format.Pod(pod))
            result.Fail(err)
            return
        }

        // If we ever allow updating a pod from non-host-network to
        // host-network, we may use a stale IP.
        if !kubecontainer.IsHostNetworkPod(pod) {
            // Overwrite the podIP passed in the pod status, since we just started the pod sandbox.
            podIP = m.determinePodSandboxIP(pod.Namespace, pod.Name, podSandboxStatus)
            glog.V(4).Infof("Determined the ip %q for pod %q after sandbox changed", podIP, format.Pod(pod))
        }
    }

和上面删除的方法一样，调用相应的容器运行时去完成操作

func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32) (string, string, error) {
    podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt)
    if err != nil {
        message := fmt.Sprintf("GeneratePodSandboxConfig for pod %q failed: %v", format.Pod(pod), err)
        glog.Error(message)
        return "", message, err
    }

    // Create pod logs directory
    err = m.osInterface.MkdirAll(podSandboxConfig.LogDirectory, 0755)
    if err != nil {
        message := fmt.Sprintf("Create pod log directory for pod %q failed: %v", format.Pod(pod), err)
        glog.Errorf(message)
        return "", message, err
    }

    podSandBoxID, err := m.runtimeService.RunPodSandbox(podSandboxConfig)
    if err != nil {
        message := fmt.Sprintf("CreatePodSandbox for pod %q failed: %v", format.Pod(pod), err)
        glog.Error(message)
        return "", message, err
    }

    return podSandBoxID, "", nil
}

如果是docker就在pkg/kubelet/dockershim/docker_sandbox.go这个里面RunPodSandbox实现
无非是先拉镜像然后启动容器，只有一个cni的部分需要注意，这个在以后的代码分析中会介绍。言归正传，创建完基础容器后

5.启动init容器

init容器是为业务容器做初始化工作的，譬如可以预先从网络上面加载一些动态资源

// Step 5: start init containers.
    status, next, done := findNextInitContainerToRun(pod, podStatus)
    if status != nil && status.ExitCode != 0 {
        // container initialization has failed, flag the pod as failed
        initContainerResult := kubecontainer.NewSyncResult(kubecontainer.InitContainer, status.Name)
        initContainerResult.Fail(kubecontainer.ErrRunInitContainer, fmt.Sprintf("init container %q exited with %d", status.Name, status.ExitCode))
        result.AddSyncResult(initContainerResult)
        if pod.Spec.RestartPolicy == v1.RestartPolicyNever {
            utilruntime.HandleError(fmt.Errorf("error running pod %q init container %q, restart=Never: %#v", format.Pod(pod), status.Name, status))
            return
        }
        utilruntime.HandleError(fmt.Errorf("Error running pod %q init container %q, restarting: %#v", format.Pod(pod), status.Name, status))
    }
    if next != nil {
        if len(podContainerChanges.ContainersToStart) == 0 {
            glog.V(4).Infof("No containers to start, stopping at init container %+v in pod %v", next.Name, format.Pod(pod))
            return
        }

        // If we need to start the next container, do so now then exit
        container := next
        startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, container.Name)
        result.AddSyncResult(startContainerResult)

        isInBackOff, msg, err := m.doBackOff(pod, container, podStatus, backOff)
        if isInBackOff {
            startContainerResult.Fail(err, msg)
            glog.V(4).Infof("Backing Off restarting init container %+v in pod %v", container, format.Pod(pod))
            return
        }

        glog.V(4).Infof("Creating init container %+v in pod %v", container, format.Pod(pod))
        if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP); err != nil {
            startContainerResult.Fail(err, msg)
            utilruntime.HandleError(fmt.Errorf("init container start failed: %v: %s", err, msg))
            return
        }

        // Successfully started the container; clear the entry in the failure
        glog.V(4).Infof("Completed init container %q for pod %q", container.Name, format.Pod(pod))
        return
    }
    if !done {
        // init container still running
        glog.V(4).Infof("An init container is still running in pod %v", format.Pod(pod))
        return
    }
    if podContainerChanges.InitFailed {
        glog.V(4).Infof("Not all init containers have succeeded for pod %v", format.Pod(pod))
        return
    }

方法通过m.startContainer启动initcontainer。

6.启动业务容器

到这一步才是启动客户自己定义的容器。

// Step 6: start containers in podContainerChanges.ContainersToStart.
    for idx := range podContainerChanges.ContainersToStart {
        container := &pod.Spec.Containers[idx]
        startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, container.Name)
        result.AddSyncResult(startContainerResult)

        isInBackOff, msg, err := m.doBackOff(pod, container, podStatus, backOff)
        if isInBackOff {
            startContainerResult.Fail(err, msg)
            glog.V(4).Infof("Backing Off restarting container %+v in pod %v", container, format.Pod(pod))
            continue
        }

        glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod))
        if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP); err != nil {
            startContainerResult.Fail(err, msg)
            utilruntime.HandleError(fmt.Errorf("container start failed: %v: %s", err, msg))
            continue
        }
    }

就是通过读取podContainerChanges.ContainersToStart管道里面，需要启动的容器，然后for循环逐一创建这个pod里面的container。
整个pod同步的代码说完了，总计6个步骤。