Docker源码分析三：启动Docker Daemon

Docker源码分析三：启动Docker Daemon
Docker Daemon是Docker架构中运行在后台的守护进程，大致可以分为Docker Server、Engine和Job部分。三者的关系大致如下：Docker Daemon通过Docker Server模块接收Docker Client的请求，并在Engine处理请求，然后根据请求类型，创建出指定的Job并运行。本文基于Docker 18.02.0-ce的源码分析Docker Daemon的启动相关内容。
Daemon的main函数位于docker-ce/components/engine/cmd/dockerd/docker.go文件中，如下：

func main() {
    if reexec.Init() {
        return
    }

    // Set terminal emulation based on platform as required.
    _, stdout, stderr := term.StdStreams()

    // @jhowardmsft - maybe there is a historic reason why on non-Windows, stderr is used
    // here. However, on Windows it makes no sense and there is no need.
    if runtime.GOOS == "windows" {
        logrus.SetOutput(stdout)
    } else {
        logrus.SetOutput(stderr)
    }
    // 创建cmd
    cmd := newDaemonCommand()
    cmd.SetOutput(stdout)
    // 执行cmd
    if err := cmd.Execute(); err != nil {
        fmt.Fprintf(stderr, "%s\n", err)
        os.Exit(1)
    }
}

func newDaemonCommand() *cobra.Command {
    opts := newDaemonOptions(config.New())

    cmd := &cobra.Command{
        Use:           "dockerd [OPTIONS]",  // 命令用法
        Short:         "A self-sufficient runtime for containers.",   // help时输出的命令提示信息
        SilenceUsage:  true,
        SilenceErrors: true,
        Args:          cli.NoArgs,
        // 命令的执行时的回调函数
        RunE: func(cmd *cobra.Command, args []string) error {
            opts.flags = cmd.Flags()
            return runDaemon(opts)  // 接受命令行参数执行启动函数 
        },
    }
    // 为根命令设置usage, help等命令及错误处理
    cli.SetupRootCommand(cmd)

    // 新建一个flag
    flags := cmd.Flags()
    // 添加－v查看版本的参数
    flags.BoolVarP(&opts.version, "version", "v", false, "Print version information and quit")
    // 添加－config-file参数
    flags.StringVar(&opts.configFile, "config-file", defaultDaemonConfigFile, "Daemon configuration file")
    opts.InstallFlags(flags)
    installConfigFlags(opts.daemonConfig, flags)
    installServiceFlags(flags)

    return cmd
}

// 命令的执行方法
func runDaemon(opts *daemonOptions) error {
    if opts.version {
        showVersion()  // 打印出版本信息
        return nil
    }
    // 新建一个daemon CLI
    daemonCli := NewDaemonCli()

    // Windows specific settings as these are not defaulted.
    if runtime.GOOS == "windows" {
        if opts.daemonConfig.Pidfile == "" {
            opts.daemonConfig.Pidfile = filepath.Join(opts.daemonConfig.Root, "docker.pid")
        }
        if opts.configFile == "" {
            opts.configFile = filepath.Join(opts.daemonConfig.Root, `config\daemon.json`)
        }
    }

    // On Windows, this may be launching as a service or with an option to
    // register the service.
    stop, runAsService, err := initService(daemonCli)
    if err != nil {
        logrus.Fatal(err)
    }

    if stop {
        return nil
    }

    // If Windows SCM manages the service - no need for PID files
    if runAsService {
        opts.daemonConfig.Pidfile = ""
    }
    // 启动daemon
    err = daemonCli.start(opts)
    notifyShutdown(err)
    return err
}

DaemonCli的结构如下：

type DaemonCli struct {
    *config.Config
    configFile *string
    flags      *pflag.FlagSet

    api             *apiserver.Server
    d               *daemon.Daemon
    authzMiddleware *authorization.Middleware // authzMiddleware enables to dynamically reload the authorization plugins
}

DaemonCli的start方法如下：

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
    stopc := make(chan bool) // 创建一个通道
    defer close(stopc)  // 函数结束时关闭通道

    // warn from uuid package when running the daemon
    uuid.Loggerf = logrus.Warnf

    opts.SetDefaultOptions(opts.flags)  // 设置默认参数

    // 从opts中将参数导入config
    if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
        return err
    }
    cli.configFile = &opts.configFile
    cli.flags = opts.flags

    if cli.Config.Debug {
        debug.Enable()
    }

    if cli.Config.Experimental {
        logrus.Warn("Running experimental build")
    }

    logrus.SetFormatter(&logrus.TextFormatter{
        TimestampFormat: jsonmessage.RFC3339NanoFixed,
        DisableColors:   cli.Config.RawLogs,
        FullTimestamp:   true,
    })

    system.InitLCOW(cli.Config.Experimental)

    if err := setDefaultUmask(); err != nil {
        return fmt.Errorf("Failed to set umask: %v", err)
    }

    if len(cli.LogConfig.Config) > 0 {
        if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
            return fmt.Errorf("Failed to set log opts: %v", err)
        }
    }

    // Create the daemon root before we create ANY other files (PID, or migrate keys)
    // to ensure the appropriate ACL is set (particularly relevant on Windows)
    // 创建daemon的root路径
    if err := daemon.CreateDaemonRoot(cli.Config); err != nil {
        return err
    }

    if cli.Pidfile != "" {
        // 创建一个Pidfile文件 /var/run/docker.pid
        pf, err := pidfile.New(cli.Pidfile)
        if err != nil {
            return fmt.Errorf("Error starting daemon: %v", err)
        }
        defer func() {  // 函数结束时清除文件
            if err := pf.Remove(); err != nil {
                logrus.Error(err)
            }
        }()
    }

    // TODO: extract to newApiServerConfig()
    serverConfig := &apiserver.Config{  // server的config文件
        Logging:     true,
        SocketGroup: cli.Config.SocketGroup,
        Version:     dockerversion.Version,
        CorsHeaders: cli.Config.CorsHeaders,
    }

    if cli.Config.TLS {
        tlsOptions := tlsconfig.Options{
            CAFile:             cli.Config.CommonTLSOptions.CAFile,
            CertFile:           cli.Config.CommonTLSOptions.CertFile,
            KeyFile:            cli.Config.CommonTLSOptions.KeyFile,
            ExclusiveRootPools: true,
        }

        if cli.Config.TLSVerify {
            // server requires and verifies client's certificate
            tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
        }
        tlsConfig, err := tlsconfig.Server(tlsOptions)
        if err != nil {
            return err
        }
        serverConfig.TLSConfig = tlsConfig
    }

    if len(cli.Config.Hosts) == 0 {
        cli.Config.Hosts = make([]string, 1)
    }
    // 新建一个server实例
    cli.api = apiserver.New(serverConfig)

    var hosts []string
    // 循环遍历配置文件里的所有host，增加server监听
    for i := 0; i < len(cli.Config.Hosts); i++ {
        var err error
        if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
            return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
        }
        // 解析协议及地址
        protoAddr := cli.Config.Hosts[i]
        protoAddrParts := strings.SplitN(protoAddr, "://", 2)
        if len(protoAddrParts) != 2 {
            return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
        }

        proto := protoAddrParts[0]
        addr := protoAddrParts[1]

        // It's a bad idea to bind to TCP without tlsverify.
        if proto == "tcp" && (serverConfig.TLSConfig == nil || serverConfig.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert) {
            logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting --tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
        }
        ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
        if err != nil {
            return err
        }
        ls = wrapListeners(proto, ls)
        // If we're binding to a TCP port, make sure that a container doesn't try to use it.
        if proto == "tcp" {
            if err := allocateDaemonPort(addr); err != nil {
                return err
            }
        }
        logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
        hosts = append(hosts, protoAddrParts[1])
        cli.api.Accept(addr, ls...)
    }
    // 新建一个default的registryserver
    registryService, err := registry.NewService(cli.Config.ServiceOptions)
    if err != nil {
        return err
    }

    rOpts, err := cli.getRemoteOptions()
    if err != nil {
        return fmt.Errorf("Failed to generate containerd options: %s", err)
    }
    containerdRemote, err := libcontainerd.New(filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), rOpts...)
    if err != nil {
        return err
    }
    signal.Trap(func() {
        cli.stop()
        <-stopc // wait for daemonCli.start() to return
    }, logrus.StandardLogger())

    // Notify that the API is active, but before daemon is set up.
    preNotifySystem()

    pluginStore := plugin.NewStore()

    if err := cli.initMiddlewares(cli.api, serverConfig, pluginStore); err != nil {
        logrus.Fatalf("Error creating middlewares: %v", err)
    }
    // 新建daemon
    d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
    if err != nil {
        return fmt.Errorf("Error starting daemon: %v", err)
    }

    d.StoreHosts(hosts)

    // validate after NewDaemon has restored enabled plugins. Dont change order.
    if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil {
        return fmt.Errorf("Error validating authorization plugin: %v", err)
    }

    // TODO: move into startMetricsServer()
    if cli.Config.MetricsAddress != "" {
        if !d.HasExperimental() {
            return fmt.Errorf("metrics-addr is only supported when experimental is enabled")
        }
        if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {
            return err
        }
    }

    // TODO: createAndStartCluster()
    name, _ := os.Hostname()

    // Use a buffered channel to pass changes from store watch API to daemon
    // A buffer allows store watch API and daemon processing to not wait for each other
    watchStream := make(chan *swarmapi.WatchMessage, 32)

    c, err := cluster.New(cluster.Config{
        Root:                   cli.Config.Root,
        Name:                   name,
        Backend:                d,
        PluginBackend:          d.PluginManager(),
        NetworkSubnetsProvider: d,
        DefaultAdvertiseAddr:   cli.Config.SwarmDefaultAdvertiseAddr,
        RuntimeRoot:            cli.getSwarmRunRoot(),
        WatchStream:            watchStream,
    })
    if err != nil {
        logrus.Fatalf("Error creating cluster component: %v", err)
    }
    d.SetCluster(c)
    err = c.Start()
    if err != nil {
        logrus.Fatalf("Error starting cluster component: %v", err)
    }

    // Restart all autostart containers which has a swarm endpoint
    // and is not yet running now that we have successfully
    // initialized the cluster.
    d.RestartSwarmContainers()

    logrus.Info("Daemon has completed initialization")

    cli.d = d

    routerOptions, err := newRouterOptions(cli.Config, d)
    if err != nil {
        return err
    }
    routerOptions.api = cli.api
    routerOptions.cluster = c

    initRouter(routerOptions)

    // process cluster change notifications
    watchCtx, cancel := context.WithCancel(context.Background())
    defer cancel()
    go d.ProcessClusterNotifications(watchCtx, watchStream)

    cli.setupConfigReloadTrap()

    // The serve API routine never exits unless an error occurs
    // We need to start it as a goroutine and wait on it so
    // daemon doesn't exit
    serveAPIWait := make(chan error)
    go cli.api.Wait(serveAPIWait)

    // after the daemon is done setting up we can notify systemd api
    notifySystem()

    // Daemon is fully initialized and handling API traffic
    // Wait for serve API to complete
    errAPI := <-serveAPIWait
    c.Cleanup()
    shutdownDaemon(d)
    containerdRemote.Cleanup()
    if errAPI != nil {
        return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
    }

    return nil
}

NewDaemon做的具体内容将在下一篇文章进行详细讲述。