func (pm *Manager) enable(p *plugin) error {
spec, err := pm.initSpec(p)
if err != nil {
return err
}
p.restartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
if err := pm.containerdClient.Create(p.P.ID, libcontainerd.Spec(*spec), libcontainerd.WithRestartManager(p.restartManager)); err != nil { // POC-only
return err
}
socket := p.P.Manifest.Interface.Socket
p.client, err = plugins.NewClient("unix://"+filepath.Join(p.runtimeSourcePath, socket), nil)
if err != nil {
return err
}
pm.Lock() // fixme: lock single record
p.P.Active = true
pm.save()
pm.Unlock()
for _, typ := range p.P.Manifest.Interface.Types {
if handler := pm.handlers[typ.String()]; handler != nil {
handler(p.Name(), p.Client())
}
}
return nil
}
func (pm *Manager) enable(p *v2.Plugin, force bool) error {
if p.IsEnabled() && !force {
return fmt.Errorf("plugin %s is already enabled", p.Name())
}
spec, err := p.InitSpec(oci.DefaultSpec(), pm.libRoot)
if err != nil {
return err
}
p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), libcontainerd.WithRestartManager(p.RestartManager)); err != nil {
if err := p.RestartManager.Cancel(); err != nil {
logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
}
return err
}
p.PClient, err = plugins.NewClient("unix://"+filepath.Join(p.RuntimeSourcePath, p.GetSocket()), nil)
if err != nil {
if err := p.RestartManager.Cancel(); err != nil {
logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
}
return err
}
pm.pluginStore.SetState(p, true)
pm.pluginStore.CallHandler(p)
return nil
}
func (pm *Manager) enable(p *plugin, force bool) error {
if p.PluginObj.Enabled && !force {
return fmt.Errorf("plugin %s is already enabled", p.Name())
}
spec, err := pm.initSpec(p)
if err != nil {
return err
}
p.restartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
if err := pm.containerdClient.Create(p.PluginObj.ID, libcontainerd.Spec(*spec), libcontainerd.WithRestartManager(p.restartManager)); err != nil { // POC-only
if err := p.restartManager.Cancel(); err != nil {
logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
}
return err
}
socket := p.PluginObj.Manifest.Interface.Socket
p.client, err = plugins.NewClient("unix://"+filepath.Join(p.runtimeSourcePath, socket), nil)
if err != nil {
if err := p.restartManager.Cancel(); err != nil {
logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
}
return err
}
pm.Lock() // fixme: lock single record
p.PluginObj.Enabled = true
pm.save()
pm.Unlock()
for _, typ := range p.PluginObj.Manifest.Interface.Types {
if handler := pm.handlers[typ.String()]; handler != nil {
handler(p.Name(), p.Client())
}
}
return nil
}
func (daemon *Daemon) restore() error {
var (
debug = utils.IsDebugEnabled()
currentDriver = daemon.GraphDriverName()
containers = make(map[string]*container.Container)
)
if !debug {
logrus.Info("Loading containers: start.")
}
dir, err := ioutil.ReadDir(daemon.repository)
if err != nil {
return err
}
containerCount := 0
for _, v := range dir {
id := v.Name()
container, err := daemon.load(id)
if !debug && logrus.GetLevel() == logrus.InfoLevel {
fmt.Print(".")
containerCount++
}
if err != nil {
logrus.Errorf("Failed to load container %v: %v", id, err)
continue
}
// Ignore the container if it does not support the current driver being used by the graph
if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver {
rwlayer, err := daemon.layerStore.GetRWLayer(container.ID)
if err != nil {
logrus.Errorf("Failed to load container mount %v: %v", id, err)
continue
}
container.RWLayer = rwlayer
logrus.Debugf("Loaded container %v", container.ID)
containers[container.ID] = container
} else {
logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID)
}
}
var migrateLegacyLinks bool
restartContainers := make(map[*container.Container]chan struct{})
activeSandboxes := make(map[string]interface{})
for _, c := range containers {
if err := daemon.registerName(c); err != nil {
logrus.Errorf("Failed to register container %s: %s", c.ID, err)
continue
}
if err := daemon.Register(c); err != nil {
logrus.Errorf("Failed to register container %s: %s", c.ID, err)
continue
}
// The LogConfig.Type is empty if the container was created before docker 1.12 with default log driver.
// We should rewrite it to use the daemon defaults.
// Fixes https://github.com/docker/docker/issues/22536
if c.HostConfig.LogConfig.Type == "" {
if err := daemon.mergeAndVerifyLogConfig(&c.HostConfig.LogConfig); err != nil {
logrus.Errorf("Failed to verify log config for container %s: %q", c.ID, err)
continue
}
}
}
var wg sync.WaitGroup
var mapLock sync.Mutex
for _, c := range containers {
wg.Add(1)
go func(c *container.Container) {
defer wg.Done()
rm := c.RestartManager(false)
if c.IsRunning() || c.IsPaused() {
if err := daemon.containerd.Restore(c.ID, libcontainerd.WithRestartManager(rm)); err != nil {
logrus.Errorf("Failed to restore %s with containerd: %s", c.ID, err)
return
}
if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() {
options, err := daemon.buildSandboxOptions(c)
if err != nil {
logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err)
}
mapLock.Lock()
activeSandboxes[c.NetworkSettings.SandboxID] = options
mapLock.Unlock()
}
}
// fixme: only if not running
// get list of containers we need to restart
if daemon.configStore.AutoRestart && !c.IsRunning() && !c.IsPaused() && c.ShouldRestart() {
mapLock.Lock()
restartContainers[c] = make(chan struct{})
mapLock.Unlock()
}
if c.RemovalInProgress {
// We probably crashed in the middle of a removal, reset
// the flag.
//
// We DO NOT remove the container here as we do not
// know if the user had requested for either the
// associated volumes, network links or both to also
// be removed. So we put the container in the "dead"
// state and leave further processing up to them.
logrus.Debugf("Resetting RemovalInProgress flag from %v", c.ID)
c.ResetRemovalInProgress()
c.SetDead()
c.ToDisk()
}
// if c.hostConfig.Links is nil (not just empty), then it is using the old sqlite links and needs to be migrated
if c.HostConfig != nil && c.HostConfig.Links == nil {
migrateLegacyLinks = true
}
}(c)
}
wg.Wait()
daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes)
if err != nil {
return fmt.Errorf("Error initializing network controller: %v", err)
}
// migrate any legacy links from sqlite
linkdbFile := filepath.Join(daemon.root, "linkgraph.db")
var legacyLinkDB *graphdb.Database
if migrateLegacyLinks {
legacyLinkDB, err = graphdb.NewSqliteConn(linkdbFile)
if err != nil {
return fmt.Errorf("error connecting to legacy link graph DB %s, container links may be lost: %v", linkdbFile, err)
}
defer legacyLinkDB.Close()
}
// Now that all the containers are registered, register the links
for _, c := range containers {
if migrateLegacyLinks {
if err := daemon.migrateLegacySqliteLinks(legacyLinkDB, c); err != nil {
return err
}
}
if err := daemon.registerLinks(c, c.HostConfig); err != nil {
logrus.Errorf("failed to register link for container %s: %v", c.ID, err)
}
}
group := sync.WaitGroup{}
for c, notifier := range restartContainers {
group.Add(1)
go func(c *container.Container, chNotify chan struct{}) {
defer group.Done()
logrus.Debugf("Starting container %s", c.ID)
// ignore errors here as this is a best effort to wait for children to be
// running before we try to start the container
children := daemon.children(c)
timeout := time.After(5 * time.Second)
for _, child := range children {
if notifier, exists := restartContainers[child]; exists {
select {
case <-notifier:
case <-timeout:
}
}
}
// Make sure networks are available before starting
daemon.waitForNetworks(c)
if err := daemon.containerStart(c); err != nil {
logrus.Errorf("Failed to start container %s: %s", c.ID, err)
}
close(chNotify)
}(c, notifier)
}
group.Wait()
// any containers that were started above would already have had this done,
// however we need to now prepare the mountpoints for the rest of the containers as well.
// This shouldn't cause any issue running on the containers that already had this run.
// This must be run after any containers with a restart policy so that containerized plugins
// can have a chance to be running before we try to initialize them.
for _, c := range containers {
// if the container has restart policy, do not
// prepare the mountpoints since it has been done on restarting.
// This is to speed up the daemon start when a restart container
// has a volume and the volume dirver is not available.
if _, ok := restartContainers[c]; ok {
continue
}
group.Add(1)
go func(c *container.Container) {
defer group.Done()
if err := daemon.prepareMountPoints(c); err != nil {
logrus.Error(err)
}
}(c)
}
group.Wait()
if !debug {
if logrus.GetLevel() == logrus.InfoLevel && containerCount > 0 {
fmt.Println()
}
logrus.Info("Loading containers: done.")
}
return nil
}
func (pm *Manager) restore(p *v2.Plugin) error {
p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
return pm.containerdClient.Restore(p.GetID(), libcontainerd.WithRestartManager(p.RestartManager))
}
func (daemon *Daemon) restore() error {
var (
debug = utils.IsDebugEnabled()
currentDriver = daemon.GraphDriverName()
containers = make(map[string]*container.Container)
)
if !debug {
logrus.Info("Loading containers: start.")
}
dir, err := ioutil.ReadDir(daemon.repository)
if err != nil {
return err
}
for _, v := range dir {
id := v.Name()
container, err := daemon.load(id)
if !debug && logrus.GetLevel() == logrus.InfoLevel {
fmt.Print(".")
}
if err != nil {
logrus.Errorf("Failed to load container %v: %v", id, err)
continue
}
// Ignore the container if it does not support the current driver being used by the graph
if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver {
rwlayer, err := daemon.layerStore.GetRWLayer(container.ID)
if err != nil {
logrus.Errorf("Failed to load container mount %v: %v", id, err)
continue
}
container.RWLayer = rwlayer
logrus.Debugf("Loaded container %v", container.ID)
containers[container.ID] = container
} else {
logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID)
}
}
var migrateLegacyLinks bool
restartContainers := make(map[*container.Container]chan struct{})
for _, c := range containers {
if err := daemon.registerName(c); err != nil {
logrus.Errorf("Failed to register container %s: %s", c.ID, err)
continue
}
if err := daemon.Register(c); err != nil {
logrus.Errorf("Failed to register container %s: %s", c.ID, err)
continue
}
}
var wg sync.WaitGroup
var mapLock sync.Mutex
for _, c := range containers {
wg.Add(1)
go func(c *container.Container) {
defer wg.Done()
rm := c.RestartManager(false)
if c.IsRunning() || c.IsPaused() {
// Fix activityCount such that graph mounts can be unmounted later
if err := daemon.layerStore.ReinitRWLayer(c.RWLayer); err != nil {
logrus.Errorf("Failed to ReinitRWLayer for %s due to %s", c.ID, err)
return
}
if err := daemon.containerd.Restore(c.ID, libcontainerd.WithRestartManager(rm)); err != nil {
logrus.Errorf("Failed to restore with containerd: %q", err)
return
}
}
// fixme: only if not running
// get list of containers we need to restart
if daemon.configStore.AutoRestart && !c.IsRunning() && !c.IsPaused() && c.ShouldRestart() {
mapLock.Lock()
restartContainers[c] = make(chan struct{})
mapLock.Unlock()
}
// if c.hostConfig.Links is nil (not just empty), then it is using the old sqlite links and needs to be migrated
if c.HostConfig != nil && c.HostConfig.Links == nil {
migrateLegacyLinks = true
}
}(c)
}
wg.Wait()
// migrate any legacy links from sqlite
linkdbFile := filepath.Join(daemon.root, "linkgraph.db")
var legacyLinkDB *graphdb.Database
if migrateLegacyLinks {
legacyLinkDB, err = graphdb.NewSqliteConn(linkdbFile)
if err != nil {
return fmt.Errorf("error connecting to legacy link graph DB %s, container links may be lost: %v", linkdbFile, err)
}
defer legacyLinkDB.Close()
}
// Now that all the containers are registered, register the links
for _, c := range containers {
if migrateLegacyLinks {
if err := daemon.migrateLegacySqliteLinks(legacyLinkDB, c); err != nil {
return err
}
}
if err := daemon.registerLinks(c, c.HostConfig); err != nil {
logrus.Errorf("failed to register link for container %s: %v", c.ID, err)
}
}
group := sync.WaitGroup{}
for c, notifier := range restartContainers {
group.Add(1)
go func(c *container.Container, chNotify chan struct{}) {
defer group.Done()
logrus.Debugf("Starting container %s", c.ID)
// ignore errors here as this is a best effort to wait for children to be
// running before we try to start the container
children := daemon.children(c)
timeout := time.After(5 * time.Second)
for _, child := range children {
if notifier, exists := restartContainers[child]; exists {
select {
case <-notifier:
case <-timeout:
}
}
}
if err := daemon.containerStart(c); err != nil {
logrus.Errorf("Failed to start container %s: %s", c.ID, err)
}
close(chNotify)
}(c, notifier)
}
group.Wait()
// any containers that were started above would already have had this done,
// however we need to now prepare the mountpoints for the rest of the containers as well.
// This shouldn't cause any issue running on the containers that already had this run.
// This must be run after any containers with a restart policy so that containerized plugins
// can have a chance to be running before we try to initialize them.
for _, c := range containers {
// if the container has restart policy, do not
// prepare the mountpoints since it has been done on restarting.
// This is to speed up the daemon start when a restart container
// has a volume and the volume dirver is not available.
if _, ok := restartContainers[c]; ok {
continue
}
group.Add(1)
go func(c *container.Container) {
defer group.Done()
if err := daemon.prepareMountPoints(c); err != nil {
logrus.Error(err)
}
}(c)
}
group.Wait()
if !debug {
if logrus.GetLevel() == logrus.InfoLevel {
fmt.Println()
}
logrus.Info("Loading containers: done.")
}
return nil
}
// containerStart prepares the container to run by setting up everything the
// container needs, such as storage and networking, as well as links
// between containers. The container is left waiting for a signal to
// begin running.
func (daemon *Daemon) containerStart(container *container.Container) (err error) {
container.Lock()
defer container.Unlock()
if container.Running {
return nil
}
if container.RemovalInProgress || container.Dead {
return fmt.Errorf("Container is marked for removal and cannot be started.")
}
// if we encounter an error during start we need to ensure that any other
// setup has been cleaned up properly
defer func() {
if err != nil {
container.SetError(err)
// if no one else has set it, make sure we don't leave it at zero
if container.ExitCode == 0 {
container.ExitCode = 128
}
container.ToDisk()
daemon.Cleanup(container)
}
}()
if err := daemon.conditionalMountOnStart(container); err != nil {
return err
}
// Make sure NetworkMode has an acceptable value. We do this to ensure
// backwards API compatibility.
container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig)
if err := daemon.initializeNetworking(container); err != nil {
return err
}
spec, err := daemon.createSpec(container)
if err != nil {
return err
}
if err := daemon.containerd.Create(container.ID, *spec, libcontainerd.WithRestartManager(container.RestartManager(true))); err != nil {
errDesc := grpc.ErrorDesc(err)
logrus.Errorf("Create container failed with error: %s", errDesc)
// if we receive an internal error from the initial start of a container then lets
// return it instead of entering the restart loop
// set to 127 for container cmd not found/does not exist)
if strings.Contains(errDesc, "executable file not found") ||
strings.Contains(errDesc, "no such file or directory") ||
strings.Contains(errDesc, "system cannot find the file specified") {
container.ExitCode = 127
}
// set to 126 for container cmd can't be invoked errors
if strings.Contains(errDesc, syscall.EACCES.Error()) {
container.ExitCode = 126
}
container.Reset(false)
return fmt.Errorf("%s", errDesc)
}
return nil
}
// containerStart prepares the container to run by setting up everything the
// container needs, such as storage and networking, as well as links
// between containers. The container is left waiting for a signal to
// begin running.
func (daemon *Daemon) containerStart(container *container.Container) (err error) {
container.Lock()
defer container.Unlock()
if container.Running {
return nil
}
if container.RemovalInProgress || container.Dead {
return fmt.Errorf("Container is marked for removal and cannot be started.")
}
// if we encounter an error during start we need to ensure that any other
// setup has been cleaned up properly
defer func() {
if err != nil {
container.SetError(err)
// if no one else has set it, make sure we don't leave it at zero
if container.ExitCode() == 0 {
container.SetExitCode(128)
}
container.ToDisk()
daemon.Cleanup(container)
// if containers AutoRemove flag is set, remove it after clean up
if container.HostConfig.AutoRemove {
container.Unlock()
if err := daemon.ContainerRm(container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
logrus.Errorf("can't remove container %s: %v", container.ID, err)
}
container.Lock()
}
}
}()
if err := daemon.conditionalMountOnStart(container); err != nil {
return err
}
// Make sure NetworkMode has an acceptable value. We do this to ensure
// backwards API compatibility.
container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig)
if err := daemon.initializeNetworking(container); err != nil {
return err
}
spec, err := daemon.createSpec(container)
if err != nil {
return err
}
createOptions := []libcontainerd.CreateOption{libcontainerd.WithRestartManager(container.RestartManager(true))}
copts, err := daemon.getLibcontainerdCreateOptions(container)
if err != nil {
return err
}
if copts != nil {
createOptions = append(createOptions, *copts...)
}
if err := daemon.containerd.Create(container.ID, *spec, createOptions...); err != nil {
errDesc := grpc.ErrorDesc(err)
logrus.Errorf("Create container failed with error: %s", errDesc)
// if we receive an internal error from the initial start of a container then lets
// return it instead of entering the restart loop
// set to 127 for container cmd not found/does not exist)
if strings.Contains(errDesc, container.Path) &&
(strings.Contains(errDesc, "executable file not found") ||
strings.Contains(errDesc, "no such file or directory") ||
strings.Contains(errDesc, "system cannot find the file specified")) {
container.SetExitCode(127)
}
// set to 126 for container cmd can't be invoked errors
if strings.Contains(errDesc, syscall.EACCES.Error()) {
container.SetExitCode(126)
}
// attempted to mount a file onto a directory, or a directory onto a file, maybe from user specified bind mounts
if strings.Contains(errDesc, syscall.ENOTDIR.Error()) {
errDesc += ": Are you trying to mount a directory onto a file (or vice-versa)? Check if the specified host path exists and is the expected type"
container.SetExitCode(127)
}
container.Reset(false)
return fmt.Errorf("%s", errDesc)
}
return nil
}
// containerStart prepares the container to run by setting up everything the
// container needs, such as storage and networking, as well as links
// between containers. The container is left waiting for a signal to
// begin running.
//容器启动的核心方法。
func (daemon *Daemon) containerStart(container *container.Container) (err error) {
//这里的锁是干什么的？
container.Lock()
defer container.Unlock()
if container.Running {
return nil
}
if container.RemovalInProgress || container.Dead {
return fmt.Errorf("Container is marked for removal and cannot be started.")
}
// if we encounter an error during start we need to ensure that any other
// setup has been cleaned up properly
defer func() {
if err != nil {
container.SetError(err)
// if no one else has set it, make sure we don't leave it at zero
if container.ExitCode == 0 {
container.ExitCode = 128
}
container.ToDisk()
daemon.Cleanup(container)
attributes := map[string]string{
"exitCode": fmt.Sprintf("%d", container.ExitCode),
}
daemon.LogContainerEventWithAttributes(container, "die", attributes)
}
}()
//挂载容器的文件系统。会调用daemon/daemon.go中的Mount()方法。
//不过那个方法比较奇怪，感觉正常情况下并不会做什么事情。
if err := daemon.conditionalMountOnStart(container); err != nil {
return err
}
// Make sure NetworkMode has an acceptable value. We do this to ensure
// backwards API compatibility.
//设置默认的网络模式。
container.HostConfig = runconfig.SetDefaultNetModeIfBlank(container.HostConfig)
/*
initializeNetworking() 对网络进行初始化，docker网络模式有三种，分别是 bridge模式
（每个容器用户单独的网络栈），host模式（与宿主机共用一个网络栈），contaier模式
（与其他容器共用一个网络栈，猜测kubernate中的pod所用的模式）；
根据config和hostConfig中的参数来确定容器的网络模式，然后调动libnetwork包来建立网络
*/
if err := daemon.initializeNetworking(container); err != nil {
return err
}
//创建容器关于namespace和cgroup等的运行环境。在daemon/oci_linux.go中。
//Spec中包括了容器的最基本的信息。
spec, err := daemon.createSpec(container)
if err != nil {
return err
}
//运行容器，详细请见libcontainerd/client_linux.go中Create()方法。
if err := daemon.containerd.Create(container.ID, *spec, libcontainerd.WithRestartManager(container.RestartManager(true))); err != nil {
// if we receive an internal error from the initial start of a container then lets
// return it instead of entering the restart loop
// set to 127 for container cmd not found/does not exist)
if strings.Contains(err.Error(), "executable file not found") ||
strings.Contains(err.Error(), "no such file or directory") ||
strings.Contains(err.Error(), "system cannot find the file specified") {
container.ExitCode = 127
err = fmt.Errorf("Container command '%s' not found or does not exist.", container.Path)
}
// set to 126 for container cmd can't be invoked errors
if strings.Contains(err.Error(), syscall.EACCES.Error()) {
container.ExitCode = 126
err = fmt.Errorf("Container command '%s' could not be invoked.", container.Path)
}
container.Reset(false)
// start event is logged even on error
daemon.LogContainerEvent(container, "start")
return err
}
return nil
}