dockerd路由和初始化

我们知道docker现在分为核心的服务端dockerd和接受用户命令的客户端docker。docker接受并解析客户端的操作指令，然后访问dockerd相应的url，让dockerd做实际的处理。dockerd实际上可简化为一个web后台，提供各种web接口API让docker访问。熟悉web开发的应该知道路由这个概念，现在我们就看看dockerd的路由以及初始化过程。
dcokerd的入口函数在文件docker/cmd/dockerd/docker.go

func main() {
    if reexec.Init() {
        return
    }

    // Set terminal emulation based on platform as required.
    _, stdout, stderr := term.StdStreams()
    logrus.SetOutput(stderr)

    cmd := newDaemonCommand()
    cmd.SetOutput(stdout)
    if err := cmd.Execute(); err != nil {
        fmt.Fprintf(stderr, "%s\n", err)
        os.Exit(1)
    }
}

main函数很简单，跟docker类似，都是解析用户，然后执行相应初始化操作。

func newDaemonCommand() *cobra.Command {
    opts := daemonOptions{
        daemonConfig: daemon.NewConfig(),
        common:       cliflags.NewCommonOptions(),//comm 选项空白对象
    }
        //根命令
    cmd := &cobra.Command{
        Use:           "dockerd [OPTIONS]",
        Short:         "A self-sufficient runtime for containers.",
        SilenceUsage:  true,
        SilenceErrors: true,
        Args:          cli.NoArgs,
        RunE: func(cmd *cobra.Command, args []string) error {
            opts.flags = cmd.Flags()
            //根命令回调函数
            return runDaemon(opts)
        },
    }
    cli.SetupRootCommand(cmd)

    flags := cmd.Flags()
    flags.BoolVarP(&opts.version, "version", "v", false, "Print version information and quit")
    //默认配置文件"/etc/docker/daemon.json"
    flags.StringVar(&opts.configFile, flagDaemonConfigFile, defaultDaemonConfigFile, "Daemon configuration file")

    //通过install，从flag获取选项值，并将相应值传到各个模块
    opts.common.InstallFlags(flags)
    opts.daemonConfig.InstallFlags(flags)
    //在linux下执行空白函数
    installServiceFlags(flags)

    return cmd
}

解析完命令就会执行函数runDaemon

func runDaemon(opts daemonOptions) error {
    if opts.version {
        showVersion()
        return nil
    }
        //空的命令行终端
    daemonCli := NewDaemonCli()

    // On Windows, this may be launching as a service or with an option to
    // register the service.
    //在linux下不会执行
    stop, err := initService(daemonCli)
    if err != nil {
        logrus.Fatal(err)
    }

    if stop {
        return nil
    }
       //启动一个server
    err = daemonCli.start(opts)
    notifyShutdown(err)
    return err
}

最终会走到err = daemonCli.start(opts)，启动一个web服务，这个函数内容比较多，关键部分有注释

func (cli *DaemonCli) start(opts daemonOptions) (err error) {
    stopc := make(chan bool)
    defer close(stopc)

    // warn from uuid package when running the daemon
    uuid.Loggerf = logrus.Warnf

    // SetDefaultOptions sets default values for options after flag parsing is
    // complete
    opts.common.SetDefaultOptions(opts.flags)

    if opts.common.TrustKey == "" {
        opts.common.TrustKey = filepath.Join(
            getDaemonConfDir(),
            cliflags.DefaultTrustKeyFile)
    }
    //载入配置文件，填充返回config
    if cli.Config, err = loadDaemonCliConfig(opts); err != nil {
        return err
    }
    cli.configFile = &opts.configFile
    //选项集
    cli.flags = opts.flags

    if cli.Config.Debug {
        utils.EnableDebug()
    }
         //实验版本
    if utils.ExperimentalBuild() {
        logrus.Warn("Running experimental build")
    }
        //设置日志格式
    logrus.SetFormatter(&logrus.TextFormatter{
        TimestampFormat: jsonlog.RFC3339NanoFixed,
        DisableColors:   cli.Config.RawLogs,
    })
        //设置文件权限
    if err := setDefaultUmask(); err != nil {
        return fmt.Errorf("Failed to set umask: %v", err)
    }
       //检查日志引擎是否存在
    if len(cli.LogConfig.Config) > 0 {
        if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
            return fmt.Errorf("Failed to set log opts: %v", err)
        }
    }
        //建立pid文件
    if cli.Pidfile != "" {
        pf, err := pidfile.New(cli.Pidfile)
        if err != nil {
            return fmt.Errorf("Error starting daemon: %v", err)
        }
        defer func() {
            if err := pf.Remove(); err != nil {
                logrus.Error(err)
            }
        }()
    }

    serverConfig := &apiserver.Config{
        Logging:     true,
        SocketGroup: cli.Config.SocketGroup,
        Version:     dockerversion.Version,
        EnableCors:  cli.Config.EnableCors,
        CorsHeaders: cli.Config.CorsHeaders,
    }

    if cli.Config.TLS {
        tlsOptions := tlsconfig.Options{
            CAFile:   cli.Config.CommonTLSOptions.CAFile,
            CertFile: cli.Config.CommonTLSOptions.CertFile,
            KeyFile:  cli.Config.CommonTLSOptions.KeyFile,
        }

        if cli.Config.TLSVerify {
            // server requires and verifies client's certificate
            tlsOptions.ClientAuth = tls.RequireAndVerifyClientCert
        }
        tlsConfig, err := tlsconfig.Server(tlsOptions)
        if err != nil {
            return err
        }
        serverConfig.TLSConfig = tlsConfig
    }
    //如果没有加-H选项参数，长度为0，一般我们直接运行，是没有加这个参数的，Host地址将使用默认的/var/run/docker.sock
//长度为0，建立一个长度为1的切片
    if len(cli.Config.Hosts) == 0 {
        //logrus.Warn("[!] DIDN'T SET HOST [!]")
        cli.Config.Hosts = make([]string, 1)
    }
        //生成一个server对象，----server包含多个http服务器
    api := apiserver.New(serverConfig)
    cli.api = api

    for i := 0; i < len(cli.Config.Hosts); i++ {
        var err error
        //默认的情况下，Host为0，进入ParseHost函数，将被赋值DefaultHost，即是 DefaultUnixSocket = "/var/run/docker.sock"，这是个unix套接字形式的地址，只能本地访问
        if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
            return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
        }
        logrus.Warn("Docker Host:"+cli.Config.Hosts[i])
        protoAddr := cli.Config.Hosts[i]
        protoAddrParts := strings.SplitN(protoAddr, "://", 2)
        if len(protoAddrParts) != 2 {
            return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
        }

        proto := protoAddrParts[0]  //unix
        addr := protoAddrParts[1] // /var/run/docker.sock

        // It's a bad idea to bind to TCP without tlsverify.
        if proto == "tcp" && (serverConfig.TLSConfig == nil || serverConfig.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert) {
            logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
        }
                //初始化地址监听，serverConfig.SocketGroup 默认docker
        ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
        if err != nil {
            return err
        }
        //logrus.Warn("Init SocketGroup:"+serverConfig.SocketGroup)

        //包装，为旧的docker客户端做兼容处理
        ls = wrapListeners(proto, ls)
        // If we're binding to a TCP port, make sure that a container doesn't try to use it.
        if proto == "tcp" {
            if err := allocateDaemonPort(addr); err != nil {
                return err
            }
        }
        logrus.Debugf("Listener created for HTTP on %s (%s)", proto, addr)
        //将监听添加到服务管理器---server
        api.Accept(addr, ls...)
    }
        //tls 相关
    if err := migrateKey(); err != nil {
        return err
    }
    // FIXME: why is this down here instead of with the other TrustKey logic above?
    cli.TrustKeyPath = opts.common.TrustKey

    //a new instance of DefaultService
    registryService := registry.NewService(cli.Config.ServiceOptions)

    //creates a fresh instance of libcontainerd remote.
    containerdRemote, err := libcontainerd.New(cli.getLibcontainerdRoot(), cli.getPlatformRemoteOptions()...)
    if err != nil {
        return err
    }
    signal.Trap(func() {
        cli.stop()
        <-stopc // wait for daemonCli.start() to return
    })
    //d为Daemon类型，实现了所有API接口，提前注意下，我们后面将分析
    d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote)
    if err != nil {
        return fmt.Errorf("Error starting daemon: %v", err)
    }

    name, _ := os.Hostname()

    c, err := cluster.New(cluster.Config{
        Root:                   cli.Config.Root,
        Name:                   name,
        Backend:                d,
        NetworkSubnetsProvider: d,
        DefaultAdvertiseAddr:   cli.Config.SwarmDefaultAdvertiseAddr,
        RuntimeRoot:            cli.getSwarmRunRoot(),
    })
    if err != nil {
        logrus.Fatalf("Error creating cluster component: %v", err)
    }

    // Restart all autostart containers which has a swarm endpoint
    // and is not yet running now that we have successfully
    // initialized the cluster.
    d.RestartSwarmContainers()

    logrus.Info("Daemon has completed initialization")

    logrus.WithFields(logrus.Fields{
        "version":     dockerversion.Version,
        "commit":      dockerversion.GitCommit,
        "graphdriver": d.GraphDriverName(),
    }).Info("Docker daemon")

    cli.initMiddlewares(api, serverConfig)
    //初始化路由
    initRouter(api, d, c)

    cli.d = d
    cli.setupConfigReloadTrap()

    // The serve API routine never exits unless an error occurs
    // We need to start it as a goroutine and wait on it so
    // daemon doesn't exit
    serveAPIWait := make(chan error)
    go api.Wait(serveAPIWait)

    // after the daemon is done setting up we can notify systemd api
    notifySystem()

    // Daemon is fully initialized and handling API traffic
    // Wait for serve API to complete
    errAPI := <-serveAPIWait
    c.Cleanup()
    shutdownDaemon(d, 15)
    containerdRemote.Cleanup()
    if errAPI != nil {
        return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
    }

    return nil
}

关键的地方两个，一个是建立socket监听，一个是路由初始化注册相关API。我们先看下建立socket监听。进入 listeners.Init
其实如果深入分析下去很简单，就是最终都是net.Listen，依赖Go的一个网络库

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
    ls := []net.Listener{}

    switch proto {
    case "fd":
        fds, err := listenFD(addr, tlsConfig)
        if err != nil {
            return nil, err
        }
        ls = append(ls, fds...)
        //如果我们通过-H选项设置一个tcp地址，就可以远程访问，这个之前提到过
    case "tcp":
        l, err := sockets.NewTCPSocket(addr, tlsConfig)
        if err != nil {
            return nil, err
        }
        ls = append(ls, l)
        //默认情况下，监听的是unix本地套接字
    case "unix":
        l, err := sockets.NewUnixSocket(addr, socketGroup)
        if err != nil {
            return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
        }
        ls = append(ls, l)
    default:
        return nil, fmt.Errorf("invalid protocol format: %q", proto)
    }

    return ls, nil
}

第二个是路由初始化，我们分析路由初始化函数initRouter(api, d, c)。可以看到很多API在这里进行注册，如果我们展开分析路由注册函数，再结合客户端docker分析，会发API路径和docker命令访问的URL是对应的。

func initRouter(s *apiserver.Server, d *daemon.Daemon, c *cluster.Cluster) {
    decoder := runconfig.ContainerDecoder{}

    routers := []router.Router{}

    // we need to add the checkpoint router before the container router or the DELETE gets masked
    routers = addExperimentalRouters(routers, d, decoder)

    routers = append(routers, []router.Router{
        container.NewRouter(d, decoder), //关于容器命令的API
        image.NewRouter(d, decoder),     //关于镜命令的API
        systemrouter.NewRouter(d, c),    //关于系命令的API api/server/router/system被重命名了
        volume.NewRouter(d),             //关于卷命令的API
        build.NewRouter(dockerfile.NewBuildManager(d)),//关于构建命令的API
        swarmrouter.NewRouter(c),
    }...)

    if d.NetworkControllerEnabled() {
        routers = append(routers, network.NewRouter(d, c))
    }

    s.InitRouter(utils.IsDebugEnabled(), routers...)
}