【docker 17 源码分析】 Docker Daemon启动

基础知识

     Daemon通过三种方式监听请求，unix，tcp，fd，默认使用unix domain socket(/var/run/Docker.sock)。对于远程请求，可以开启tcp socket(-H tcp:0.0.0.0:2375)，或者固定IP(-H tcp://192.168.0.1:2375)。可以使用多种配置如下：

       $ sudo dockerd -H unix:///var/run/docker.sock -H tcp://192.168.59.106 -H tcp://10.10.10.2

     docker为docker client 和 docker daemon，client端发送命令，daemon端负责执行client发送过来的命令（获取和存储镜像、管理容器等）。两者可以通过TCP，HTTP和UNIX SOCKET来进行通信

• 创建 Docker 运行环境
• httpserver 服务于Docker Client，接收并处理相应请求

一. Docker Daemon 启动源码分析

     入口代码为cmd/dockerd/docker.go。

func main() {
       if reexec.Init() {
              return
       }

       _, stdout, stderr := term.StdStreams()
       logrus.SetOutput(stderr)

       cmd := newDaemonCommand()
       cmd.SetOutput(stdout)
       ......
}

     1.1 reexec 是 docker 自己实现的一个 package，https://groups.google.com/forum/#!topic/docker-dev/ePLDji_qBvE 给的解释可能是个过期的代码

     1.2 logrus 第三方的 log 模块，initLogging 就是将输出定向到 stderr。

     1.3 newDaemonCommand

func newDaemonCommand() *cobra.Command {
       opts := daemonOptions{
              daemonConfig: config.New(),
              common:       cliflags.NewCommonOptions(),
       }

       cmd := &cobra.Command{
              Use:           "dockerd [OPTIONS]",
              Short:         "A self-sufficient runtime for containers.",
              SilenceUsage:  true,
              SilenceErrors: true,
              Args:          cli.NoArgs,
              RunE: func(cmd *cobra.Command, args []string) error {
                     opts.flags = cmd.Flags()
                     return runDaemon(opts)
              },
       }
       ......

       return cmd
}

1.3.1 dockerd 启动 deamon 实际运行的是 runDeamon 函数，可以使用 docker -D 启动，进入 runDeamon()后，

初始化 daemonCli，执行 start 方法，runDeamon 中除了 api service 还会 d, err := daemon.NewDaemon(cli.Config, registryService,

containerdRemote)，注册 registry，和 contanerd 等。

1.3.2defaultDaemonConfigFile默认配置文件"/etc/docker/daemon.json"

func runDaemon(opts daemonOptions) error {

       daemonCli := NewDaemonCli()

       ......

       err = daemonCli.start(opts)
       notifyShutdown(err)
       return err
}

type DaemonCli struct {
       *config.Config
       configFile *string
       flags      *pflag.FlagSet

       api             *apiserver.Server
       d               *daemon.Daemon
       authzMiddleware *authorization.Middleware
}

    1.4  daemonCli.start cli.Pidfile 为 /var/run/docker.pid，创建文件并写入 pid

if cli.Pidfile != "" {
       pf, err := pidfile.New(cli.Pidfile)
       if err != nil {
              return fmt.Errorf("Error starting daemon: %v", err)
       }
       defer func() {
              if err := pf.Remove(); err != nil {
                     logrus.Error(err)
              }
       }()
}

     1.5  daemonCli.start 启动一个server

• len(cli.Config.Hosts)，如果没有-H参数，长度为0，默认使用的是 /var/run/docker.sock。make一个长度为1的切片
• api server.New(serverConfig) 生成一个api server 对象，包含多个http服务

func (cli *DaemonCli) start(opts daemonOptions) (err error) {
       ......

       serverConfig := &apiserver.Config{
              Logging:     true,
              SocketGroup: cli.Config.SocketGroup,
              Version:     dockerversion.Version,
              EnableCors:  cli.Config.EnableCors,
              CorsHeaders: cli.Config.CorsHeaders,
       }


       if len(cli.Config.Hosts) == 0 {
              cli.Config.Hosts = make([]string, 1)
       }

       api := apiserver.New(serverConfig)
       cli.api = api

}

    1.6  daemonCli.start 初始化监听地址，因为启动没有指定任何 host, ParseHost 设置为默认值，unix:///var/run/docker.sock，proto 为 unix，addr 为 /var/run/docker.sock，listeners.Init 创建 socket server，accept 客户端发来的请求。其他的 TCP 协议等一样的分析。

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
       ......

       for i := 0; i < len(cli.Config.Hosts); i++ {
              var err error
              if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
                     return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
              }

              protoAddr := cli.Config.Hosts[i]
              protoAddrParts := strings.SplitN(protoAddr, "://", 2)
              if len(protoAddrParts) != 2 {
                     return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
              }

              proto := protoAddrParts[0]
              addr := protoAddrParts[1]
              ......
              ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
              if err != nil {
                     return err
              }
              ls = wrapListeners(proto, ls)
              ......
              hosts = append(hosts, protoAddrParts[1])
              cli.api.Accept(addr, ls...)
       }

// Accept sets a listener the server accepts connections into.
func (s *Server) Accept(addr string, listeners ...net.Listener) {
       for _, listener := range listeners {
              httpServer := &HTTPServer{
                     srv: &http.Server{
                            Addr: addr,
                     },
                     l: listener,
              }
              s.servers = append(s.servers, httpServer)
       }
}

    1.7  daemonCli.start 生成一个 DefaultService 结构体：

registryService := registry.NewService(cli.Config.ServiceOptions)

// DefaultService is a registry service. It tracks configuration data such as a list
// of mirrors.
type DefaultService struct {
       config *serviceConfig
       mu     sync.Mutex
}

// NewService returns a new instance of DefaultService ready to be
// installed into an engine.
func NewService(options ServiceOptions) *DefaultService {
       return &DefaultService{
              config: newServiceConfig(options),
       }
}

    1.8  daemonCli.start 生成一个 libcontainerd remote 实例：创建目录 /var/run/docker/libcontainerd，目录下 rpc addr 为 docker-containerd.sock

// New creates a fresh instance of libcontainerd remote.
func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
       r := &remote{
              stateDir:    stateDir,
              daemonPid:   -1,
              eventTsPath: filepath.Join(stateDir, eventTimestampFilename),
       }
       for _, option := range options {
              if err := option.Apply(r); err != nil {
                     return nil, err
              }
       }

       if err := sysinfo.MkdirAll(stateDir, 0700); err != nil {

       if r.rpcAddr == "" {
              r.rpcAddr = filepath.Join(stateDir, containerdSockFilename)
       }

       if r.startDaemon {
              if err := r.runContainerdDaemon(); err != nil {

       return r, nil
}

     1.8.1 runContainerDaemon，pid 文件为 /var/run/docker/libcontainerd/docker-containerd.pid，打开文件查看先前进程是否存在，否则启动一个新的实例，命令大致为：docker-containerd -l unix:///var/run/docker/libcontainerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd --shim docker-containerd-shim --runtime docker-runc

写入 pid 至文件。setOOMScore 写入 /proc/${pid}/oom_score_adj

func (r *remote) runContainerdDaemon() error {
       pidFilename := filepath.Join(r.stateDir, containerdPidFilename)
       f, err := os.OpenFile(pidFilename, os.O_RDWR|os.O_CREATE, 0600)

       if n > 0 {
              pid, err := strconv.ParseUint(string(b[:n]), 10, 64)
       
              if system.IsProcessAlive(int(pid)) {
                     logrus.Infof("libcontainerd: previous instance of containerd still alive (%d)", pid)
                     r.daemonPid = int(pid)
                     return nil
              }
       }

       // Start a new instance
       args := []string{
              "-l", fmt.Sprintf("unix://%s", r.rpcAddr),
              "--metrics-interval=0",
              "--start-timeout", "2m",
              "--state-dir", filepath.Join(r.stateDir, containerdStateDir),
       }
       ......

       cmd := exec.Command(containerdBinary, args...)
       if err := cmd.Start(); err != nil {
  
       r.daemonPid = cmd.Process.Pid
       return nil
}

    1.8.1.1 执行可以查看 ps axf | grep docker，启动进程 docker-containerd

    1.8.2 启动完 docker-containerd 进程，使用 rpc 连接到这个进程，

func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
       dialOpts := []grpc.DialOption{
              grpc.WithInsecure(),
              grpc.WithBackoffMaxDelay(2 * time.Second),
              grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
                     return net.DialTimeout("unix", addr, timeout)
              }),
       }
       conn, err := grpc.Dial(r.rpcAddr, dialOpts...)

       r.rpcConn = conn
       r.apiClient = containerd.NewAPIClient(conn)

       // Get the timestamp to restore from
       t := r.getLastEventTimestamp()
       tsp, err := ptypes.TimestampProto(t)
     
       r.restoreFromTimestamp = tsp

       go r.handleConnectionChange()

       if err := r.startEventsMonitor(); err != nil {

       return r, nil
}

    handleConnectionChange 启动 for 循环，500 毫秒一次进行检查。

    1.9 daemon.NewDaemon 放入第二章节详解

d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
if err != nil {
       return fmt.Errorf("Error starting daemon: %v", err)
}

    1.10 initRouter(routerOptions)，路由有 checkoutpoint，container，image，volume 等。POST DELETE GET 操作

func initRouter(opts routerOptions) {
       decoder := runconfig.ContainerDecoder{}

       routers := []router.Router{
              // we need to add the checkpoint router before the container router or the DELETE gets masked
              checkpointrouter.NewRouter(opts.daemon, decoder),
              container.NewRouter(opts.daemon, decoder),
              image.NewRouter(opts.daemon, decoder),
              systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),
              volume.NewRouter(opts.daemon),
              build.NewRouter(opts.buildBackend, opts.daemon),
              sessionrouter.NewRouter(opts.sessionManager),
              swarmrouter.NewRouter(opts.cluster),
              pluginrouter.NewRouter(opts.daemon.PluginManager()),
              distributionrouter.NewRouter(opts.daemon),
       }

       if opts.daemon.NetworkControllerEnabled() {
              routers = append(routers, network.NewRouter(opts.daemon, opts.cluster))
       }

       if opts.daemon.HasExperimental() {
              for _, r := range routers {
                     for _, route := range r.Routes() {
                            if experimental, ok := route.(router.ExperimentalRoute); ok {
                                   experimental.Enable()
                            }
                     }
              }
       }

       opts.api.InitRouter(routers...)
}

          1.10.1 initRouter 主要的是 InitRouter 函数实用的是如下：

func (s *Server) InitRouter(routers ...router.Router) {
       s.routers = append(s.routers, routers...)

       m := s.createMux()
       s.routerSwapper = &routerSwapper{
              router: m,
       }
}

// createMux initializes the main router the server uses.
func (s *Server) createMux() *mux.Router {
       m := mux.NewRouter()

       for _, apiRouter := range s.routers {
              for _, r := range apiRouter.Routes() {
                     f := s.makeHTTPHandler(r.Handler())

                     m.Path(versionMatcher + r.Path()).Methods(r.Method()).Handler(f)
                     m.Path(r.Path()).Methods(r.Method()).Handler(f)
              }
       }

       debugRouter := debug.NewRouter()
       s.routers = append(s.routers, debugRouter)
       for _, r := range debugRouter.Routes() {
              f := s.makeHTTPHandler(r.Handler())
              m.Path("/debug" + r.Path()).Handler(f)
       }

       err := errors.NewRequestNotFoundError(fmt.Errorf("page not found"))
       notFoundHandler := httputils.MakeErrorHandler(err)
       m.HandleFunc(versionMatcher+"/{path:.*}", notFoundHandler)
       m.NotFoundHandler = notFoundHandler

       return m
}

         1.11 cli.api.Wait 等待请求在 /var/run/docker.sock

func (s *Server) Wait(waitChan chan error) {
       if err := s.serveAPI(); err != nil {
              logrus.Errorf("ServeAPI error: %v", err)
              waitChan <- err
              return
       }
       waitChan <- nil
}

docker 启动总结：

    大部分是参数的检查

    启动 API server 监听请求

    建立 API 路由

    启动进程 libcontainerd，

    镜像之间关系等

    

二. NewDaemon 函数源码分析

        位置：daemon/daemon.go

setDefaultMtu(config)

        2.1 设置 MTU。若 config 中有值则使用 config 中的，否则设置为默认的 1500

func verifyDaemonSettings(conf *config.Config) error {
       // Check for mutually incompatible config options
       if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {
              return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")
       }
       if !conf.BridgeConfig.EnableIPTables && !conf.BridgeConfig.InterContainerCommunication {
              return fmt.Errorf("You specified --iptables=false with --icc=false. ICC=false uses iptables to function. Please set --icc or --iptables to true")
       }
       if !conf.BridgeConfig.EnableIPTables && conf.BridgeConfig.EnableIPMasq {
              conf.BridgeConfig.EnableIPMasq = false
       }
       if err := VerifyCgroupDriver(conf); err != nil {
              return err
       }
       if conf.CgroupParent != "" && UsingSystemd(conf) {
              if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") {
                     return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
              }
       }

       if conf.DefaultRuntime == "" {
              conf.DefaultRuntime = config.StockRuntimeName
       }
       if conf.Runtimes == nil {
              conf.Runtimes = make(map[string]types.Runtime)
       }
       conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary}

       return nil
}

      2.2 验证 daemon 的配置

• 不能同时指定网桥和网桥的 IP。如果指定网桥，应使用网桥的当前IP地址，不能再设置 IP 地址。
• 不能同时禁用 iptable 和 icc(容器间通信)。如果禁用 icc，docker 会在宿主机的 iptables 的 FORWARD chain 中添加一条 docker 容器间流量均 DROP 的规则，此时设置 EnableIPTables 为f alse，冲突!!!

func isBridgeNetworkDisabled(conf *config.Config) bool {
       return conf.BridgeConfig.Iface == config.DisableNetworkBridge
}

    2.3 设置是否启用网桥。disableNetworkBridge 为 none，Iface 为空，所以 DisableBridge 为false

if !platformSupported {
       return nil, errSystemNotSupported
}

if err := checkSystem(); err != nil {
       return nil, err
}

   2.4 检查系统支持和用户权限。需要 root 权限，linux 中 uid = 0 为 root 用户，检查内核版本

uidMaps, gidMaps, err := setupRemappedRoot(config)

rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)

   2.5 docker 支持 user namespace，主要隔离安全相关 identifiers 和 attributes，包括用户 ID、用户组 ID、root 目录、key 以及特殊权限。普通用户的进程通过 clone() 创建的新进程在新 user namespace 中拥有不同的用户和用户组，这意味着一个进程在容器外属于一个没有特权的普通用户，但是创建的容器进程却属于拥有所有权限的超级用户。

// setupDaemonProcess sets various settings for the daemon's process
func setupDaemonProcess(config *config.Config) error {
       // setup the daemons oom_score_adj
       return setupOOMScoreAdj(config.OOMScoreAdjust)
}

   2.6 设置OOM killer值。oom killer设置/proc/self/oom_score_adj，值的范围为–17~15。正值易被OOM Killer选定。如果设置为–17，表示禁止被kill掉

tmp, err := prepareTempDir(config.Root, rootIDs)

realTmp, err := getRealPath(tmp)

os.Setenv("TMPDIR", realTmp)

   2.7  prepareTempDir 设置 tmp dir。如有环境变量 DOCKER_TMPDIR，则使用这个，否则创建 /var/lib/docker/tmp，将旧的变为 /var/lib/docker/tmp-old。getRealPath 然后创建一个指向tmp文件的符号链接realTmp，并把 realTmp 赋值给环境变量TMPDIR

// configureMaxThreads sets the Go runtime max threads threshold
// which is 90% of the kernel setting from /proc/sys/kernel/threads-max
func configureMaxThreads(config *config.Config) error {
       mt, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")
       if err != nil {
              return err
       }
       mtint, err := strconv.Atoi(strings.TrimSpace(string(mt)))
       if err != nil {
              return err
       }
       maxThreads := (mtint / 100) * 90
       debug.SetMaxThreads(maxThreads)
       logrus.Debugf("Golang's threads limit set to %d", maxThreads)
       return nil
}

     2.8  设置最大线程数。从 /proc/sys/kernel/threads-max 文件读取值，然后乘以0.9

daemonRepo := filepath.Join(config.Root, "containers")
if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
       return nil, err
}

   2.9  创建容器目录，位于 docker daemon 目录下的 containers。daemon 创建容器会把容器的元数据信息放此

driverName := os.Getenv("DOCKER_DRIVER")
if driverName == "" {
       driverName = config.GraphDriver
}

d.stores[runtime.GOOS] = daemonStore{graphDriver: driverName}
d.RegistryService = registryServiced.PluginStore = pluginStore// Plugin system initialization should happen before restore. Do not change order.d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{       Root:               filepath.Join(config.Root, "plugins"),       ExecRoot:           getPluginExecRoot(config.Root),       Store:              d.PluginStore,       Executor:           containerdRemote,       RegistryService:    registryService,       LiveRestoreEnabled: config.LiveRestoreEnabled,       LogPluginEvent:     d.LogPluginEvent, // todo: make private       AuthzMiddleware:    config.AuthzMiddleware,})if err != nil {       return nil, errors.Wrap(err, "couldn't create plugin manager")}d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{       StorePath:                 config.Root,       MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),       GraphDriver:               driverName,       GraphDriverOptions:        config.GraphOptions,       UIDMaps:                   uidMaps,       GIDMaps:                   gidMaps,       PluginGetter:              d.PluginStore,       ExperimentalEnabled:       config.Experimental,})if err != nil {       return nil, err}

     2.10 填充 daemon 结构体

// Plugin system initialization should happen before restore. Do not change order.
d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
       Root:               filepath.Join(config.Root, "plugins"),
       ExecRoot:           getPluginExecRoot(config.Root),
       Store:              d.PluginStore,
       Executor:           containerdRemote,
       RegistryService:    registryService,
       LiveRestoreEnabled: config.LiveRestoreEnabled,
       LogPluginEvent:     d.LogPluginEvent, // todo: make private
       AuthzMiddleware:    config.AuthzMiddleware,
})
if err != nil {
       return nil, errors.Wrap(err, "couldn't create plugin manager")
}

    2.11 结构体如下 2.11.1 所示，Root 路径为 /var/lib/docker/plugins，ExecRoot 为 /run/docker/plugins，NewManager 生成一个plugin manager，如下 2.11.2 所示      

    2.11.1 结构体 ManagerConfig如下所示：

// ManagerConfig defines configuration needed to start new manager.
type ManagerConfig struct {
       Store              *Store // remove
       Executor           libcontainerd.Remote
       RegistryService    registry.Service
       LiveRestoreEnabled bool // TODO: remove
       LogPluginEvent     eventLogger
       Root               string
       ExecRoot           string
       AuthzMiddleware    *authorization.Middleware
}

     2.11.2 NewManager 函数主要是生成目录 /var/lib/docker/plugins，/run/docker/plugins，/var/lib/docker/plugins/tmp，/var/lib/docker/plugins/storage/blobs/tmp ，并初始化结构体 Manager，如下所示：

func NewManager(config ManagerConfig) (*Manager, error) {
       if config.RegistryService != nil {
              config.RegistryService = pluginRegistryService{config.RegistryService}
       }
       manager := &Manager{
              config: config,
       }
       if err := os.MkdirAll(manager.config.Root, 0700); err != nil {
       
       if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {
       
       if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {
         
       var err error
       manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct
       
       manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))
      
       manager.cMap = make(map[*v2.Plugin]*controller)

       manager.publisher = pubsub.NewPublisher(0, 0)
       return manager, nil
}

    

    2.12 如下所示，StoreOptions 结构体如下 2.12.1 所示，NewStoreFromOptions 函数 2.12.2 所示：

for platform, ds := range d.stores {
       ls, err := layer.NewStoreFromOptions(layer.StoreOptions{
              StorePath:                 config.Root,
              MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
              GraphDriver:               ds.graphDriver,
              GraphDriverOptions:        config.GraphOptions,
              IDMappings:                idMappings,
              PluginGetter:              d.PluginStore,
              ExperimentalEnabled:       config.Experimental,
              Platform:                  platform,
       })
       
       ds.graphDriver = ls.DriverName() // As layerstore may set the driver
       ds.layerStore = ls
       d.stores[platform] = ds
       graphDrivers = append(graphDrivers, ls.DriverName())
}

    2.12.1 StoreOptions 结构体如下所示：

// StoreOptions are the options used to create a new Store instance
type StoreOptions struct {
       StorePath                 string
       MetadataStorePathTemplate string
       GraphDriver               string
       GraphDriverOptions        []string
       IDMappings                *idtools.IDMappings
       PluginGetter              plugingetter.PluginGetter
       ExperimentalEnabled       bool
       Platform                  string
}

     2.12.2 NewStoreFromOptions 函数中的 driver 为 aufs，创建目录 /var/lib/docker/images/aufs/layerdb，如下所示：

// NewStoreFromOptions creates a new Store instance
func NewStoreFromOptions(options StoreOptions) (Store, error) {
       driver, err := graphdriver.New(options.GraphDriver, options.PluginGetter, graphdriver.Options{
              Root:                options.StorePath,
              DriverOptions:       options.GraphDriverOptions,
              UIDMaps:             options.IDMappings.UIDs(),
              GIDMaps:             options.IDMappings.GIDs(),
              ExperimentalEnabled: options.ExperimentalEnabled,
       })
       if err != nil {
              return nil, fmt.Errorf("error initializing graphdriver: %v", err)
       }
       logrus.Debugf("Using graph driver %s", driver)

       fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))
       if err != nil {
              return nil, err
       }

       return NewStoreFromGraphDriver(fms, driver, options.Platform)
}

d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads)

d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)

    2.13 创建上传下载管理器。设置了上传/下载的最大并发数

for platform, ds := range d.stores {
       imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)
       ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
  
       var is image.Store
       is, err = image.NewImageStore(ifs, platform, ds.layerStore)

       ds.imageRoot = imageRoot
       ds.imageStore = is
       d.stores[platform] = ds
}

    2.14 创建镜像存储目录并restore镜像。存储镜像的根目录为 /var/lib/docker/image/aufs/imagedb，然后还会在 imagedb 目录下新建两个目录，content/sha256和metadata/sha256，存储镜像内容和镜像元数据

func (daemon *Daemon) configureVolumes(rootUID, rootGID int) (*store.VolumeStore, error) {
       volumesDriver, err := local.New(daemon.configStore.Root, rootUID, rootGID)
       if err != nil {
              return nil, err
       }

       volumedrivers.RegisterPluginGetter(daemon.PluginStore)

       if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) {
              return nil, errors.New("local volume driver could not be registered")
       }
       return store.New(daemon.configStore.Root)
}

    2.14 配置 volumes。 设置数据卷 driver，数据卷是容器之间进行数据共享的一种手段；数据卷可以是一个本机命令（通过-v 标识挂载在到容器的某个目录下） 数据卷也可以作为数据卷容器，通过--volumes-from挂载到某个容器中；

    local.New 创建目录 /var/lib/docker/volumes，容器的 volume 是可从宿主机上挂载到容器内的特定目录。一个 volume 可以被多个容器挂载共享数据。volumes 以插件形式，这里的 local.New 是创建了默认的 volume driver 的实现

trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath)
if err != nil {
       return nil, err
}

     2.15 创建 key，路径在/etc/docker/key.json

trustDir := filepath.Join(config.Root, "trust")

if err := system.MkdirAll(trustDir, 0700, ""); err != nil {
       return nil, err
}

      2.17 创建 trust 目录，路径在/var/lib/docker/trust

// New returns new *Events instance
func New() *Events {
       return &Events{
              events: make([]eventtypes.Message, 0, eventsLimit),
              pub:    pubsub.NewPublisher(100*time.Millisecond, bufferSize),
       }
}

     2.18 创建event实例，产生的event通过订阅的方式获取

// NewReferenceStore creates a new reference store, tied to a file path where
// the set of references are serialized in JSON format.
func NewReferenceStore(jsonPath string) (Store, error) {
       abspath, err := filepath.Abs(jsonPath)
       if err != nil {
              return nil, err
       }

       store := &store{
              jsonPath:            abspath,
              Repositories:        make(map[string]repository),
              referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
       }
       // Load the json file if it exists, otherwise create it.
       if err := store.reload(); os.IsNotExist(err) {
              if err := store.save(); err != nil {
                     return nil, err
              }
       } else if err != nil {
              return nil, err
       }
       return store, nil
}

type store struct {
       mu sync.RWMutex
       // jsonPath is the path to the file where the serialized tag data is
       // stored.
       jsonPath string
       // Repositories is a map of repositories, indexed by name.
       Repositories map[string]repository
       // referencesByIDCache is a cache of references indexed by ID, to speed
       // up References.
       referencesByIDCache map[digest.Digest]map[string]reference.Named
}

    2.19  创建TagStore。用于管理存储镜像的仓库列表，Repositories记录了镜像仓库的映射，
referencesByIDCache记录了镜像ID和镜像全名的映射

// initDiscovery initializes the discovery watcher for this daemon.
func (daemon *Daemon) initDiscovery(conf *config.Config) error {
       advertise, err := config.ParseClusterAdvertiseSettings(conf.ClusterStore, conf.ClusterAdvertise)
       if err != nil {
              if err == discovery.ErrDiscoveryDisabled {
                     return nil
              }
              return err
       }

       conf.ClusterAdvertise = advertise
       discoveryWatcher, err := discovery.Init(conf.ClusterStore, conf.ClusterAdvertise, conf.ClusterOpts)
       if err != nil {
              return fmt.Errorf("discovery initialization failed (%v)", err)
       }

       daemon.discoveryWatcher = discoveryWatcher
       return nil
}

    2.20  服务发现。docker自带的服务发现，docker集群中使用，维护心跳、服务注册等

sysInfo := sysinfo.New(false)
// Check if Devices cgroup is mounted, it is hard requirement for container security,
// on Linux.
if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled {
       return nil, errors.New("Devices cgroup isn't mounted")
}

    2.21  获取系统信息。包括cgroup、网络配置(网桥/iptables)和linux安全(AppArmor/Seccomp)等。如下所示：

type SysInfo struct {
       // Whether the kernel supports AppArmor or not
       AppArmor bool
       // Whether the kernel supports Seccomp or not
       Seccomp bool

       cgroupMemInfo
       cgroupCPUInfo
       cgroupBlkioInfo
       cgroupCpusetInfo
       cgroupPids

       // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work
       IPv4ForwardingDisabled bool

       // Whether bridge-nf-call-iptables is supported or not
       BridgeNFCallIPTablesDisabled bool

       // Whether bridge-nf-call-ip6tables is supported or not
       BridgeNFCallIP6TablesDisabled bool

       // Whether the cgroup has the mountpoint of "devices" or not
       CgroupDevicesEnabled bool
}

d.ID = trustKey.PublicKey().KeyID()
d.repository = daemonRepo
d.containers = container.NewMemoryStore()
d.execCommands = exec.NewStore()
d.referenceStore = referenceStore
d.distributionMetadataStore = distributionMetadataStore
d.trustKey = trustKey
d.idIndex = truncindex.NewTruncIndex([]string{})
d.statsCollector = d.newStatsCollector(1 * time.Second)
d.defaultLogConfig = containertypes.LogConfig{
       Type:   config.LogConfig.Type,
       Config: config.LogConfig.Config,
}
d.EventsService = eventsService
d.volumes = volStore
d.root = config.Root
d.uidMaps = uidMaps
d.gidMaps = gidMaps
d.seccompEnabled = sysInfo.Seccomp
d.apparmorEnabled = sysInfo.AppArmor

d.nameIndex = registrar.NewRegistrar()
d.linkIndex = newLinkIndex()
d.containerdRemote = containerdRemote

    2.22  填充daemon结构体。

• container.NewMemoryStore() 创建内存store，把container的名字和container的映射存储在map中
• exec.NewStore() 创建commands的store，把commands和config的映射，用于执行容器里面的一些任务
• truncindex.NewTruncIndex([]string{}) 对contaniner或者image进行操作，截取id的一部分即可进行操作
• d.newStatsCollector(1 * time.Second) 1s收集一次运行的容器的状态，包括网络和cgroup状态。

       CPU： /proc/stat

func (daemon *Daemon) restore() error {
       containers := make(map[string]*container.Container)

       logrus.Info("Loading containers: start.")

       dir, err := ioutil.ReadDir(daemon.repository)
       if err != nil {
              return err
       }

       for _, v := range dir {
              id := v.Name()
              container, err := daemon.load(id)

2.24  restore 函数。从 /var/lib/docker/containers 读取所有目录，下面的文件夹的名字就是容器的id，存入 containers map[string]*container.Container 信息并注册如 daemon 结构体中。包括 start remove 操作 containers

         注册容器。注册信息都在内存中，启动daemon时需要重新进行注册

NewDaemon 总结：

      验证参数，image store，containers  等创建，读取 containers目录并重启等，填充 daemon 结构体。