func (pm *Manager) enable(p *plugin) error { spec, err := pm.initSpec(p) if err != nil { return err } p.restartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0) if err := pm.containerdClient.Create(p.P.ID, libcontainerd.Spec(*spec), libcontainerd.WithRestartManager(p.restartManager)); err != nil { // POC-only return err } socket := p.P.Manifest.Interface.Socket p.client, err = plugins.NewClient("unix://"+filepath.Join(p.runtimeSourcePath, socket), nil) if err != nil { return err } pm.Lock() // fixme: lock single record p.P.Active = true pm.save() pm.Unlock() for _, typ := range p.P.Manifest.Interface.Types { if handler := pm.handlers[typ.String()]; handler != nil { handler(p.Name(), p.Client()) } } return nil }
func (pm *Manager) enable(p *v2.Plugin, force bool) error { if p.IsEnabled() && !force { return fmt.Errorf("plugin %s is already enabled", p.Name()) } spec, err := p.InitSpec(oci.DefaultSpec(), pm.libRoot) if err != nil { return err } p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0) if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), libcontainerd.WithRestartManager(p.RestartManager)); err != nil { if err := p.RestartManager.Cancel(); err != nil { logrus.Errorf("enable: restartManager.Cancel failed due to %v", err) } return err } p.PClient, err = plugins.NewClient("unix://"+filepath.Join(p.RuntimeSourcePath, p.GetSocket()), nil) if err != nil { if err := p.RestartManager.Cancel(); err != nil { logrus.Errorf("enable: restartManager.Cancel failed due to %v", err) } return err } pm.pluginStore.SetState(p, true) pm.pluginStore.CallHandler(p) return nil }
func (pm *Manager) enable(p *plugin, force bool) error { if p.PluginObj.Enabled && !force { return fmt.Errorf("plugin %s is already enabled", p.Name()) } spec, err := pm.initSpec(p) if err != nil { return err } p.restartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0) if err := pm.containerdClient.Create(p.PluginObj.ID, libcontainerd.Spec(*spec), libcontainerd.WithRestartManager(p.restartManager)); err != nil { // POC-only if err := p.restartManager.Cancel(); err != nil { logrus.Errorf("enable: restartManager.Cancel failed due to %v", err) } return err } socket := p.PluginObj.Manifest.Interface.Socket p.client, err = plugins.NewClient("unix://"+filepath.Join(p.runtimeSourcePath, socket), nil) if err != nil { if err := p.restartManager.Cancel(); err != nil { logrus.Errorf("enable: restartManager.Cancel failed due to %v", err) } return err } pm.Lock() // fixme: lock single record p.PluginObj.Enabled = true pm.save() pm.Unlock() for _, typ := range p.PluginObj.Manifest.Interface.Types { if handler := pm.handlers[typ.String()]; handler != nil { handler(p.Name(), p.Client()) } } return nil }
func (daemon *Daemon) restore() error { var ( debug = utils.IsDebugEnabled() currentDriver = daemon.GraphDriverName() containers = make(map[string]*container.Container) ) if !debug { logrus.Info("Loading containers: start.") } dir, err := ioutil.ReadDir(daemon.repository) if err != nil { return err } containerCount := 0 for _, v := range dir { id := v.Name() container, err := daemon.load(id) if !debug && logrus.GetLevel() == logrus.InfoLevel { fmt.Print(".") containerCount++ } if err != nil { logrus.Errorf("Failed to load container %v: %v", id, err) continue } // Ignore the container if it does not support the current driver being used by the graph if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver { rwlayer, err := daemon.layerStore.GetRWLayer(container.ID) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue } container.RWLayer = rwlayer logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = container } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } } var migrateLegacyLinks bool restartContainers := make(map[*container.Container]chan struct{}) activeSandboxes := make(map[string]interface{}) for _, c := range containers { if err := daemon.registerName(c); err != nil { logrus.Errorf("Failed to register container %s: %s", c.ID, err) continue } if err := daemon.Register(c); err != nil { logrus.Errorf("Failed to register container %s: %s", c.ID, err) continue } // The LogConfig.Type is empty if the container was created before docker 1.12 with default log driver. // We should rewrite it to use the daemon defaults. // Fixes https://github.com/docker/docker/issues/22536 if c.HostConfig.LogConfig.Type == "" { if err := daemon.mergeAndVerifyLogConfig(&c.HostConfig.LogConfig); err != nil { logrus.Errorf("Failed to verify log config for container %s: %q", c.ID, err) continue } } } var wg sync.WaitGroup var mapLock sync.Mutex for _, c := range containers { wg.Add(1) go func(c *container.Container) { defer wg.Done() rm := c.RestartManager(false) if c.IsRunning() || c.IsPaused() { if err := daemon.containerd.Restore(c.ID, libcontainerd.WithRestartManager(rm)); err != nil { logrus.Errorf("Failed to restore %s with containerd: %s", c.ID, err) return } if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() { options, err := daemon.buildSandboxOptions(c) if err != nil { logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err) } mapLock.Lock() activeSandboxes[c.NetworkSettings.SandboxID] = options mapLock.Unlock() } } // fixme: only if not running // get list of containers we need to restart if daemon.configStore.AutoRestart && !c.IsRunning() && !c.IsPaused() && c.ShouldRestart() { mapLock.Lock() restartContainers[c] = make(chan struct{}) mapLock.Unlock() } if c.RemovalInProgress { // We probably crashed in the middle of a removal, reset // the flag. // // We DO NOT remove the container here as we do not // know if the user had requested for either the // associated volumes, network links or both to also // be removed. So we put the container in the "dead" // state and leave further processing up to them. logrus.Debugf("Resetting RemovalInProgress flag from %v", c.ID) c.ResetRemovalInProgress() c.SetDead() c.ToDisk() } // if c.hostConfig.Links is nil (not just empty), then it is using the old sqlite links and needs to be migrated if c.HostConfig != nil && c.HostConfig.Links == nil { migrateLegacyLinks = true } }(c) } wg.Wait() daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes) if err != nil { return fmt.Errorf("Error initializing network controller: %v", err) } // migrate any legacy links from sqlite linkdbFile := filepath.Join(daemon.root, "linkgraph.db") var legacyLinkDB *graphdb.Database if migrateLegacyLinks { legacyLinkDB, err = graphdb.NewSqliteConn(linkdbFile) if err != nil { return fmt.Errorf("error connecting to legacy link graph DB %s, container links may be lost: %v", linkdbFile, err) } defer legacyLinkDB.Close() } // Now that all the containers are registered, register the links for _, c := range containers { if migrateLegacyLinks { if err := daemon.migrateLegacySqliteLinks(legacyLinkDB, c); err != nil { return err } } if err := daemon.registerLinks(c, c.HostConfig); err != nil { logrus.Errorf("failed to register link for container %s: %v", c.ID, err) } } group := sync.WaitGroup{} for c, notifier := range restartContainers { group.Add(1) go func(c *container.Container, chNotify chan struct{}) { defer group.Done() logrus.Debugf("Starting container %s", c.ID) // ignore errors here as this is a best effort to wait for children to be // running before we try to start the container children := daemon.children(c) timeout := time.After(5 * time.Second) for _, child := range children { if notifier, exists := restartContainers[child]; exists { select { case <-notifier: case <-timeout: } } } // Make sure networks are available before starting daemon.waitForNetworks(c) if err := daemon.containerStart(c); err != nil { logrus.Errorf("Failed to start container %s: %s", c.ID, err) } close(chNotify) }(c, notifier) } group.Wait() // any containers that were started above would already have had this done, // however we need to now prepare the mountpoints for the rest of the containers as well. // This shouldn't cause any issue running on the containers that already had this run. // This must be run after any containers with a restart policy so that containerized plugins // can have a chance to be running before we try to initialize them. for _, c := range containers { // if the container has restart policy, do not // prepare the mountpoints since it has been done on restarting. // This is to speed up the daemon start when a restart container // has a volume and the volume dirver is not available. if _, ok := restartContainers[c]; ok { continue } group.Add(1) go func(c *container.Container) { defer group.Done() if err := daemon.prepareMountPoints(c); err != nil { logrus.Error(err) } }(c) } group.Wait() if !debug { if logrus.GetLevel() == logrus.InfoLevel && containerCount > 0 { fmt.Println() } logrus.Info("Loading containers: done.") } return nil }
func (pm *Manager) restore(p *v2.Plugin) error { p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0) return pm.containerdClient.Restore(p.GetID(), libcontainerd.WithRestartManager(p.RestartManager)) }
func (daemon *Daemon) restore() error { var ( debug = utils.IsDebugEnabled() currentDriver = daemon.GraphDriverName() containers = make(map[string]*container.Container) ) if !debug { logrus.Info("Loading containers: start.") } dir, err := ioutil.ReadDir(daemon.repository) if err != nil { return err } for _, v := range dir { id := v.Name() container, err := daemon.load(id) if !debug && logrus.GetLevel() == logrus.InfoLevel { fmt.Print(".") } if err != nil { logrus.Errorf("Failed to load container %v: %v", id, err) continue } // Ignore the container if it does not support the current driver being used by the graph if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver { rwlayer, err := daemon.layerStore.GetRWLayer(container.ID) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue } container.RWLayer = rwlayer logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = container } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } } var migrateLegacyLinks bool restartContainers := make(map[*container.Container]chan struct{}) for _, c := range containers { if err := daemon.registerName(c); err != nil { logrus.Errorf("Failed to register container %s: %s", c.ID, err) continue } if err := daemon.Register(c); err != nil { logrus.Errorf("Failed to register container %s: %s", c.ID, err) continue } } var wg sync.WaitGroup var mapLock sync.Mutex for _, c := range containers { wg.Add(1) go func(c *container.Container) { defer wg.Done() rm := c.RestartManager(false) if c.IsRunning() || c.IsPaused() { // Fix activityCount such that graph mounts can be unmounted later if err := daemon.layerStore.ReinitRWLayer(c.RWLayer); err != nil { logrus.Errorf("Failed to ReinitRWLayer for %s due to %s", c.ID, err) return } if err := daemon.containerd.Restore(c.ID, libcontainerd.WithRestartManager(rm)); err != nil { logrus.Errorf("Failed to restore with containerd: %q", err) return } } // fixme: only if not running // get list of containers we need to restart if daemon.configStore.AutoRestart && !c.IsRunning() && !c.IsPaused() && c.ShouldRestart() { mapLock.Lock() restartContainers[c] = make(chan struct{}) mapLock.Unlock() } // if c.hostConfig.Links is nil (not just empty), then it is using the old sqlite links and needs to be migrated if c.HostConfig != nil && c.HostConfig.Links == nil { migrateLegacyLinks = true } }(c) } wg.Wait() // migrate any legacy links from sqlite linkdbFile := filepath.Join(daemon.root, "linkgraph.db") var legacyLinkDB *graphdb.Database if migrateLegacyLinks { legacyLinkDB, err = graphdb.NewSqliteConn(linkdbFile) if err != nil { return fmt.Errorf("error connecting to legacy link graph DB %s, container links may be lost: %v", linkdbFile, err) } defer legacyLinkDB.Close() } // Now that all the containers are registered, register the links for _, c := range containers { if migrateLegacyLinks { if err := daemon.migrateLegacySqliteLinks(legacyLinkDB, c); err != nil { return err } } if err := daemon.registerLinks(c, c.HostConfig); err != nil { logrus.Errorf("failed to register link for container %s: %v", c.ID, err) } } group := sync.WaitGroup{} for c, notifier := range restartContainers { group.Add(1) go func(c *container.Container, chNotify chan struct{}) { defer group.Done() logrus.Debugf("Starting container %s", c.ID) // ignore errors here as this is a best effort to wait for children to be // running before we try to start the container children := daemon.children(c) timeout := time.After(5 * time.Second) for _, child := range children { if notifier, exists := restartContainers[child]; exists { select { case <-notifier: case <-timeout: } } } if err := daemon.containerStart(c); err != nil { logrus.Errorf("Failed to start container %s: %s", c.ID, err) } close(chNotify) }(c, notifier) } group.Wait() // any containers that were started above would already have had this done, // however we need to now prepare the mountpoints for the rest of the containers as well. // This shouldn't cause any issue running on the containers that already had this run. // This must be run after any containers with a restart policy so that containerized plugins // can have a chance to be running before we try to initialize them. for _, c := range containers { // if the container has restart policy, do not // prepare the mountpoints since it has been done on restarting. // This is to speed up the daemon start when a restart container // has a volume and the volume dirver is not available. if _, ok := restartContainers[c]; ok { continue } group.Add(1) go func(c *container.Container) { defer group.Done() if err := daemon.prepareMountPoints(c); err != nil { logrus.Error(err) } }(c) } group.Wait() if !debug { if logrus.GetLevel() == logrus.InfoLevel { fmt.Println() } logrus.Info("Loading containers: done.") } return nil }
// containerStart prepares the container to run by setting up everything the // container needs, such as storage and networking, as well as links // between containers. The container is left waiting for a signal to // begin running. func (daemon *Daemon) containerStart(container *container.Container) (err error) { container.Lock() defer container.Unlock() if container.Running { return nil } if container.RemovalInProgress || container.Dead { return fmt.Errorf("Container is marked for removal and cannot be started.") } // if we encounter an error during start we need to ensure that any other // setup has been cleaned up properly defer func() { if err != nil { container.SetError(err) // if no one else has set it, make sure we don't leave it at zero if container.ExitCode == 0 { container.ExitCode = 128 } container.ToDisk() daemon.Cleanup(container) } }() if err := daemon.conditionalMountOnStart(container); err != nil { return err } // Make sure NetworkMode has an acceptable value. We do this to ensure // backwards API compatibility. container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig) if err := daemon.initializeNetworking(container); err != nil { return err } spec, err := daemon.createSpec(container) if err != nil { return err } if err := daemon.containerd.Create(container.ID, *spec, libcontainerd.WithRestartManager(container.RestartManager(true))); err != nil { errDesc := grpc.ErrorDesc(err) logrus.Errorf("Create container failed with error: %s", errDesc) // if we receive an internal error from the initial start of a container then lets // return it instead of entering the restart loop // set to 127 for container cmd not found/does not exist) if strings.Contains(errDesc, "executable file not found") || strings.Contains(errDesc, "no such file or directory") || strings.Contains(errDesc, "system cannot find the file specified") { container.ExitCode = 127 } // set to 126 for container cmd can't be invoked errors if strings.Contains(errDesc, syscall.EACCES.Error()) { container.ExitCode = 126 } container.Reset(false) return fmt.Errorf("%s", errDesc) } return nil }
// containerStart prepares the container to run by setting up everything the // container needs, such as storage and networking, as well as links // between containers. The container is left waiting for a signal to // begin running. func (daemon *Daemon) containerStart(container *container.Container) (err error) { container.Lock() defer container.Unlock() if container.Running { return nil } if container.RemovalInProgress || container.Dead { return fmt.Errorf("Container is marked for removal and cannot be started.") } // if we encounter an error during start we need to ensure that any other // setup has been cleaned up properly defer func() { if err != nil { container.SetError(err) // if no one else has set it, make sure we don't leave it at zero if container.ExitCode() == 0 { container.SetExitCode(128) } container.ToDisk() daemon.Cleanup(container) // if containers AutoRemove flag is set, remove it after clean up if container.HostConfig.AutoRemove { container.Unlock() if err := daemon.ContainerRm(container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { logrus.Errorf("can't remove container %s: %v", container.ID, err) } container.Lock() } } }() if err := daemon.conditionalMountOnStart(container); err != nil { return err } // Make sure NetworkMode has an acceptable value. We do this to ensure // backwards API compatibility. container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig) if err := daemon.initializeNetworking(container); err != nil { return err } spec, err := daemon.createSpec(container) if err != nil { return err } createOptions := []libcontainerd.CreateOption{libcontainerd.WithRestartManager(container.RestartManager(true))} copts, err := daemon.getLibcontainerdCreateOptions(container) if err != nil { return err } if copts != nil { createOptions = append(createOptions, *copts...) } if err := daemon.containerd.Create(container.ID, *spec, createOptions...); err != nil { errDesc := grpc.ErrorDesc(err) logrus.Errorf("Create container failed with error: %s", errDesc) // if we receive an internal error from the initial start of a container then lets // return it instead of entering the restart loop // set to 127 for container cmd not found/does not exist) if strings.Contains(errDesc, container.Path) && (strings.Contains(errDesc, "executable file not found") || strings.Contains(errDesc, "no such file or directory") || strings.Contains(errDesc, "system cannot find the file specified")) { container.SetExitCode(127) } // set to 126 for container cmd can't be invoked errors if strings.Contains(errDesc, syscall.EACCES.Error()) { container.SetExitCode(126) } // attempted to mount a file onto a directory, or a directory onto a file, maybe from user specified bind mounts if strings.Contains(errDesc, syscall.ENOTDIR.Error()) { errDesc += ": Are you trying to mount a directory onto a file (or vice-versa)? Check if the specified host path exists and is the expected type" container.SetExitCode(127) } container.Reset(false) return fmt.Errorf("%s", errDesc) } return nil }
// containerStart prepares the container to run by setting up everything the // container needs, such as storage and networking, as well as links // between containers. The container is left waiting for a signal to // begin running. //容器启动的核心方法。 func (daemon *Daemon) containerStart(container *container.Container) (err error) { //这里的锁是干什么的? container.Lock() defer container.Unlock() if container.Running { return nil } if container.RemovalInProgress || container.Dead { return fmt.Errorf("Container is marked for removal and cannot be started.") } // if we encounter an error during start we need to ensure that any other // setup has been cleaned up properly defer func() { if err != nil { container.SetError(err) // if no one else has set it, make sure we don't leave it at zero if container.ExitCode == 0 { container.ExitCode = 128 } container.ToDisk() daemon.Cleanup(container) attributes := map[string]string{ "exitCode": fmt.Sprintf("%d", container.ExitCode), } daemon.LogContainerEventWithAttributes(container, "die", attributes) } }() //挂载容器的文件系统。会调用daemon/daemon.go中的Mount()方法。 //不过那个方法比较奇怪,感觉正常情况下并不会做什么事情。 if err := daemon.conditionalMountOnStart(container); err != nil { return err } // Make sure NetworkMode has an acceptable value. We do this to ensure // backwards API compatibility. //设置默认的网络模式。 container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig) /* initializeNetworking() 对网络进行初始化,docker网络模式有三种,分别是 bridge模式 (每个容器用户单独的网络栈),host模式(与宿主机共用一个网络栈),contaier模式 (与其他容器共用一个网络栈,猜测kubernate中的pod所用的模式); 根据config和hostConfig中的参数来确定容器的网络模式,然后调动libnetwork包来建立网络 */ if err := daemon.initializeNetworking(container); err != nil { return err } //创建容器关于namespace和cgroup等的运行环境。在daemon/oci_linux.go中。 //Spec中包括了容器的最基本的信息。 spec, err := daemon.createSpec(container) if err != nil { return err } //运行容器,详细请见libcontainerd/client_linux.go中Create()方法。 if err := daemon.containerd.Create(container.ID, *spec, libcontainerd.WithRestartManager(container.RestartManager(true))); err != nil { // if we receive an internal error from the initial start of a container then lets // return it instead of entering the restart loop // set to 127 for container cmd not found/does not exist) if strings.Contains(err.Error(), "executable file not found") || strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "system cannot find the file specified") { container.ExitCode = 127 err = fmt.Errorf("Container command '%s' not found or does not exist.", container.Path) } // set to 126 for container cmd can't be invoked errors if strings.Contains(err.Error(), syscall.EACCES.Error()) { container.ExitCode = 126 err = fmt.Errorf("Container command '%s' could not be invoked.", container.Path) } container.Reset(false) // start event is logged even on error daemon.LogContainerEvent(container, "start") return err } return nil }