func (svc *IService) remove(notify chan<- int) { defer close(notify) ctr, err := docker.FindContainer(svc.name()) if err == docker.ErrNoSuchContainer { return } else if err != nil { glog.Errorf("Could not get isvc container %s", svc.Name) return } // report the log output if output, err := exec.Command("docker", "logs", "--tail", "1000", ctr.ID).CombinedOutput(); err != nil { glog.Warningf("Could not get logs for container %s", ctr.Name) } else { glog.V(1).Infof("Exited isvc %s:\n %s", svc.Name, string(output)) } // kill the container if it is running if ctr.IsRunning() { glog.Warningf("isvc %s is still running; killing", svc.Name) ctr.Kill() } // get the exit code rc, _ := ctr.Wait(time.Second) defer func() { notify <- rc }() // delete the container if err := ctr.Delete(true); err != nil { glog.Errorf("Could not remove isvc %s: %s", ctr.Name, err) } }
// StopService terminates a particular service instance (serviceState) on the localhost. func (a *HostAgent) StopService(state *servicestate.ServiceState) error { if state == nil || state.DockerID == "" { return errors.New("missing Docker ID") } ctr, err := docker.FindContainer(state.DockerID) if err != nil { return err } return ctr.Stop(45 * time.Second) }
func (svc *IService) Exec(command []string) error { ctr, err := docker.FindContainer(svc.name()) if err != nil { return err } output, err := utils.AttachAndRun(ctr.ID, command) if err != nil { return err } os.Stdout.Write(output) return nil }
func (svc *IService) stop() error { ctr, err := docker.FindContainer(svc.name()) if err == docker.ErrNoSuchContainer { svc.setStoppedHealthStatus(nil) return nil } else if err != nil { glog.Errorf("Could not get isvc container %s", svc.Name) svc.setStoppedHealthStatus(err) return err } glog.Warningf("Stopping isvc container %s", svc.name()) err = ctr.Stop(45 * time.Second) svc.setStoppedHealthStatus(err) return err }
func (svc *IService) attach() (*docker.Container, error) { ctr, _ := docker.FindContainer(svc.name()) if ctr != nil { notify := make(chan int, 1) if !ctr.IsRunning() { glog.Infof("isvc %s found but not running; removing container %s", svc.name(), ctr.ID) go svc.remove(notify) } else if !svc.checkvolumes(ctr) { glog.Infof("isvc %s found but volumes are missing or incomplete; removing container %s", svc.name(), ctr.ID) ctr.OnEvent(docker.Die, func(cid string) { svc.remove(notify) }) svc.stop() } else { glog.Infof("Attaching to isvc %s at %s", svc.name(), ctr.ID) return ctr, nil } <-notify } glog.Infof("Creating a new container for isvc %s", svc.name()) return svc.create() }
// AttachService attempts to attach to a running container func (a *HostAgent) AttachService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error { ctr, err := docker.FindContainer(state.DockerID) if err != nil { return err } if !ctr.IsRunning() { defer exited(state.ID) return nil } ctr.OnEvent(docker.Die, func(cid string) { defer exited(state.ID) glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID) state.DockerID = cid a.removeInstance(state.ID, ctr) }) go a.setProxy(svc, ctr) return nil }
func (svc *IService) stats(halt <-chan struct{}) { registry := metrics.NewRegistry() tc := time.Tick(10 * time.Second) for { select { case <-halt: glog.Infof("stop collecting stats for %s", svc.Name) return case t := <-tc: ctr, err := docker.FindContainer(svc.name()) if err != nil { glog.Warningf("Could not find container for isvc %s: %s", svc.Name, err) break } if cpuacctStat, err := cgroup.ReadCpuacctStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Cpuacct)); err != nil { glog.Warningf("Could not read CpuacctStat for isvc %s: %s", svc.Name, err) break } else { metrics.GetOrRegisterGauge("CpuacctStat.system", registry).Update(cpuacctStat.System) metrics.GetOrRegisterGauge("CpuacctStat.user", registry).Update(cpuacctStat.User) } if memoryStat, err := cgroup.ReadMemoryStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Memory)); err != nil { glog.Warningf("Could not read MemoryStat for isvc %s: %s", svc.Name, err) break } else { metrics.GetOrRegisterGauge("cgroup.memory.pgmajfault", registry).Update(memoryStat.Pgfault) metrics.GetOrRegisterGauge("cgroup.memory.totalrss", registry).Update(memoryStat.TotalRss) metrics.GetOrRegisterGauge("cgroup.memory.cache", registry).Update(memoryStat.Cache) } // Gather the stats stats := []containerStat{} registry.Each(func(name string, i interface{}) { if metric, ok := i.(metrics.Gauge); ok { tagmap := make(map[string]string) tagmap["isvcname"] = svc.Name stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap}) } if metricf64, ok := i.(metrics.GaugeFloat64); ok { tagmap := make(map[string]string) tagmap["isvcname"] = svc.Name stats = append(stats, containerStat{name, strconv.FormatFloat(metricf64.Value(), 'f', -1, 32), t.Unix(), tagmap}) } }) // Post the stats. data, err := json.Marshal(stats) if err != nil { glog.Warningf("Error marshalling isvc stats json.") break } req, err := http.NewRequest("POST", "http://127.0.0.1:4242/api/put", bytes.NewBuffer(data)) if err != nil { glog.Warningf("Error creating isvc stats request.") break } resp, err := http.DefaultClient.Do(req) if err != nil { glog.V(4).Infof("Error making isvc stats request.") break } if strings.Contains(resp.Status, "204 No Content") == false { glog.Warningf("Couldn't post stats:", resp.Status) break } } } }
// StartService starts a new instance of the specified service and updates the control center state accordingly. func (a *HostAgent) StartService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error { glog.V(2).Infof("About to start service %s with name %s", svc.ID, svc.Name) client, err := NewControlClient(a.master) if err != nil { glog.Errorf("Could not start ControlPlane client %v", err) return err } defer client.Close() // start from a known good state if state.DockerID != "" { if ctr, err := docker.FindContainer(state.DockerID); err != nil { glog.Errorf("Could not find container %s for %s", state.DockerID, state.ID) } else if err := ctr.Delete(true); err != nil { glog.Errorf("Could not delete container %s for %s", state.DockerID, state.ID) } } // create the docker client Config and HostConfig structures necessary to create and start the service config, hostconfig, err := configureContainer(a, client, svc, state, a.virtualAddressSubnet) if err != nil { glog.Errorf("can't configure container: %v", err) return err } cjson, _ := json.MarshalIndent(config, "", " ") glog.V(3).Infof(">>> CreateContainerOptions:\n%s", string(cjson)) hcjson, _ := json.MarshalIndent(hostconfig, "", " ") glog.V(3).Infof(">>> HostConfigOptions:\n%s", string(hcjson)) cd := &docker.ContainerDefinition{ dockerclient.CreateContainerOptions{Name: state.ID, Config: config}, *hostconfig, } ctr, err := docker.NewContainer(cd, false, 10*time.Second, nil, nil) if err != nil { glog.Errorf("Error trying to create container %v: %v", config, err) return err } var started sync.WaitGroup started.Add(1) ctr.OnEvent(docker.Start, func(cid string) { glog.Infof("Instance %s (%s) for %s (%s) has started", state.ID, ctr.ID, svc.Name, svc.ID) started.Done() }) ctr.OnEvent(docker.Die, func(cid string) { defer exited(state.ID) glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID) state.DockerID = cid a.removeInstance(state.ID, ctr) }) if err := ctr.Start(time.Hour); err != nil { glog.Errorf("Could not start service state %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err) a.removeInstance(state.ID, ctr) return err } started.Wait() if err := updateInstance(state, ctr); err != nil { glog.Errorf("Could not update instance %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err) ctr.Stop(45 * time.Second) return err } go a.setProxy(svc, ctr) return nil }
// Get the state of the docker container given the dockerId func getDockerState(dockerID string) (*docker.Container, error) { glog.V(1).Infof("Inspecting container: %s", dockerID) return docker.FindContainer(dockerID) }
// Commit will merge a container into existing services' image func (dfs *DistributedFilesystem) Commit(dockerID string) (string, error) { // get the container ctr, err := docker.FindContainer(dockerID) if err != nil { glog.Errorf("Could not get container %s: %s", dockerID, err) return "", err } // verify the container is not currently running if ctr.IsRunning() { err := fmt.Errorf("cannot commit a running container") glog.Errorf("Error committing container %s: %s", ctr.ID, err) return "", err } // find the image to commit (ctr.Config.Image is the repotag) image, err := docker.FindImage(ctr.Config.Image, false) if err != nil { glog.Errorf("Could not find image %s from %s: %s", ctr.Config.Image, dockerID, err) return "", err } // verify the container is not stale (ctr.Image is the UUID) if !image.ID.IsLatest() || image.UUID != ctr.Image { err := fmt.Errorf("cannot commit a stale container") glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err) return "", err } // verify the tenantID tenantID, err := dfs.facade.GetTenantID(datastore.Get(), image.ID.User) if err != nil { glog.Errorf("Could not look up tenant %s from image %s for container %s: %s", image.ID.User, image.ID, dockerID, err) return "", err } else if tenantID != image.ID.User { err := fmt.Errorf("service is not the tenant") glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err) return "", err } // check the number of image layers if layers, err := image.History(); err != nil { glog.Errorf("Could not check history for image %s: %s", image.ID, err) return "", err } else if numLayers := len(layers); numLayers >= layer.WARN_LAYER_COUNT { glog.Warningf("Image %s has %d layers and is approaching the maximum (%d). Please squash image layers.", image.ID, numLayers, layer.MAX_LAYER_COUNT) } else { glog.V(3).Infof("Image %s has %d layers", image.ID, numLayers) } // commit the container to the image and tag newImage, err := ctr.Commit(image.ID.BaseName()) if err != nil { glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err) return "", err } // desynchronize any running containers if err := dfs.desynchronize(newImage); err != nil { glog.Warningf("Could not denote all desynchronized services: %s", err) } // snapshot the filesystem and images snapshotID, err := dfs.Snapshot(tenantID) if err != nil { glog.Errorf("Could not create a snapshot of the new image %s: %s", tenantID, err) return "", err } return snapshotID, nil }