func (svc *IService) remove(notify chan<- int) {
defer close(notify)
ctr, err := docker.FindContainer(svc.name())
if err == docker.ErrNoSuchContainer {
return
} else if err != nil {
glog.Errorf("Could not get isvc container %s", svc.Name)
return
}
// report the log output
if output, err := exec.Command("docker", "logs", "--tail", "1000", ctr.ID).CombinedOutput(); err != nil {
glog.Warningf("Could not get logs for container %s", ctr.Name)
} else {
glog.V(1).Infof("Exited isvc %s:\n %s", svc.Name, string(output))
}
// kill the container if it is running
if ctr.IsRunning() {
glog.Warningf("isvc %s is still running; killing", svc.Name)
ctr.Kill()
}
// get the exit code
rc, _ := ctr.Wait(time.Second)
defer func() { notify <- rc }()
// delete the container
if err := ctr.Delete(true); err != nil {
glog.Errorf("Could not remove isvc %s: %s", ctr.Name, err)
}
}
// StopService terminates a particular service instance (serviceState) on the localhost.
func (a *HostAgent) StopService(state *servicestate.ServiceState) error {
if state == nil || state.DockerID == "" {
return errors.New("missing Docker ID")
}
ctr, err := docker.FindContainer(state.DockerID)
if err != nil {
return err
}
return ctr.Stop(45 * time.Second)
}
func (svc *IService) Exec(command []string) error {
ctr, err := docker.FindContainer(svc.name())
if err != nil {
return err
}
output, err := utils.AttachAndRun(ctr.ID, command)
if err != nil {
return err
}
os.Stdout.Write(output)
return nil
}
func (svc *IService) stop() error {
ctr, err := docker.FindContainer(svc.name())
if err == docker.ErrNoSuchContainer {
svc.setStoppedHealthStatus(nil)
return nil
} else if err != nil {
glog.Errorf("Could not get isvc container %s", svc.Name)
svc.setStoppedHealthStatus(err)
return err
}
glog.Warningf("Stopping isvc container %s", svc.name())
err = ctr.Stop(45 * time.Second)
svc.setStoppedHealthStatus(err)
return err
}
func (svc *IService) attach() (*docker.Container, error) {
ctr, _ := docker.FindContainer(svc.name())
if ctr != nil {
notify := make(chan int, 1)
if !ctr.IsRunning() {
glog.Infof("isvc %s found but not running; removing container %s", svc.name(), ctr.ID)
go svc.remove(notify)
} else if !svc.checkvolumes(ctr) {
glog.Infof("isvc %s found but volumes are missing or incomplete; removing container %s", svc.name(), ctr.ID)
ctr.OnEvent(docker.Die, func(cid string) { svc.remove(notify) })
svc.stop()
} else {
glog.Infof("Attaching to isvc %s at %s", svc.name(), ctr.ID)
return ctr, nil
}
<-notify
}
glog.Infof("Creating a new container for isvc %s", svc.name())
return svc.create()
}
// AttachService attempts to attach to a running container
func (a *HostAgent) AttachService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
ctr, err := docker.FindContainer(state.DockerID)
if err != nil {
return err
}
if !ctr.IsRunning() {
defer exited(state.ID)
return nil
}
ctr.OnEvent(docker.Die, func(cid string) {
defer exited(state.ID)
glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
state.DockerID = cid
a.removeInstance(state.ID, ctr)
})
go a.setProxy(svc, ctr)
return nil
}
func (svc *IService) stats(halt <-chan struct{}) {
registry := metrics.NewRegistry()
tc := time.Tick(10 * time.Second)
for {
select {
case <-halt:
glog.Infof("stop collecting stats for %s", svc.Name)
return
case t := <-tc:
ctr, err := docker.FindContainer(svc.name())
if err != nil {
glog.Warningf("Could not find container for isvc %s: %s", svc.Name, err)
break
}
if cpuacctStat, err := cgroup.ReadCpuacctStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Cpuacct)); err != nil {
glog.Warningf("Could not read CpuacctStat for isvc %s: %s", svc.Name, err)
break
} else {
metrics.GetOrRegisterGauge("CpuacctStat.system", registry).Update(cpuacctStat.System)
metrics.GetOrRegisterGauge("CpuacctStat.user", registry).Update(cpuacctStat.User)
}
if memoryStat, err := cgroup.ReadMemoryStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Memory)); err != nil {
glog.Warningf("Could not read MemoryStat for isvc %s: %s", svc.Name, err)
break
} else {
metrics.GetOrRegisterGauge("cgroup.memory.pgmajfault", registry).Update(memoryStat.Pgfault)
metrics.GetOrRegisterGauge("cgroup.memory.totalrss", registry).Update(memoryStat.TotalRss)
metrics.GetOrRegisterGauge("cgroup.memory.cache", registry).Update(memoryStat.Cache)
}
// Gather the stats
stats := []containerStat{}
registry.Each(func(name string, i interface{}) {
if metric, ok := i.(metrics.Gauge); ok {
tagmap := make(map[string]string)
tagmap["isvcname"] = svc.Name
stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
}
if metricf64, ok := i.(metrics.GaugeFloat64); ok {
tagmap := make(map[string]string)
tagmap["isvcname"] = svc.Name
stats = append(stats, containerStat{name, strconv.FormatFloat(metricf64.Value(), 'f', -1, 32), t.Unix(), tagmap})
}
})
// Post the stats.
data, err := json.Marshal(stats)
if err != nil {
glog.Warningf("Error marshalling isvc stats json.")
break
}
req, err := http.NewRequest("POST", "http://127.0.0.1:4242/api/put", bytes.NewBuffer(data))
if err != nil {
glog.Warningf("Error creating isvc stats request.")
break
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
glog.V(4).Infof("Error making isvc stats request.")
break
}
if strings.Contains(resp.Status, "204 No Content") == false {
glog.Warningf("Couldn't post stats:", resp.Status)
break
}
}
}
}
// StartService starts a new instance of the specified service and updates the control center state accordingly.
func (a *HostAgent) StartService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
glog.V(2).Infof("About to start service %s with name %s", svc.ID, svc.Name)
client, err := NewControlClient(a.master)
if err != nil {
glog.Errorf("Could not start ControlPlane client %v", err)
return err
}
defer client.Close()
// start from a known good state
if state.DockerID != "" {
if ctr, err := docker.FindContainer(state.DockerID); err != nil {
glog.Errorf("Could not find container %s for %s", state.DockerID, state.ID)
} else if err := ctr.Delete(true); err != nil {
glog.Errorf("Could not delete container %s for %s", state.DockerID, state.ID)
}
}
// create the docker client Config and HostConfig structures necessary to create and start the service
config, hostconfig, err := configureContainer(a, client, svc, state, a.virtualAddressSubnet)
if err != nil {
glog.Errorf("can't configure container: %v", err)
return err
}
cjson, _ := json.MarshalIndent(config, "", " ")
glog.V(3).Infof(">>> CreateContainerOptions:\n%s", string(cjson))
hcjson, _ := json.MarshalIndent(hostconfig, "", " ")
glog.V(3).Infof(">>> HostConfigOptions:\n%s", string(hcjson))
cd := &docker.ContainerDefinition{
dockerclient.CreateContainerOptions{Name: state.ID, Config: config},
*hostconfig,
}
ctr, err := docker.NewContainer(cd, false, 10*time.Second, nil, nil)
if err != nil {
glog.Errorf("Error trying to create container %v: %v", config, err)
return err
}
var started sync.WaitGroup
started.Add(1)
ctr.OnEvent(docker.Start, func(cid string) {
glog.Infof("Instance %s (%s) for %s (%s) has started", state.ID, ctr.ID, svc.Name, svc.ID)
started.Done()
})
ctr.OnEvent(docker.Die, func(cid string) {
defer exited(state.ID)
glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
state.DockerID = cid
a.removeInstance(state.ID, ctr)
})
if err := ctr.Start(time.Hour); err != nil {
glog.Errorf("Could not start service state %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
a.removeInstance(state.ID, ctr)
return err
}
started.Wait()
if err := updateInstance(state, ctr); err != nil {
glog.Errorf("Could not update instance %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
ctr.Stop(45 * time.Second)
return err
}
go a.setProxy(svc, ctr)
return nil
}
// Get the state of the docker container given the dockerId
func getDockerState(dockerID string) (*docker.Container, error) {
glog.V(1).Infof("Inspecting container: %s", dockerID)
return docker.FindContainer(dockerID)
}
// Commit will merge a container into existing services' image
func (dfs *DistributedFilesystem) Commit(dockerID string) (string, error) {
// get the container
ctr, err := docker.FindContainer(dockerID)
if err != nil {
glog.Errorf("Could not get container %s: %s", dockerID, err)
return "", err
}
// verify the container is not currently running
if ctr.IsRunning() {
err := fmt.Errorf("cannot commit a running container")
glog.Errorf("Error committing container %s: %s", ctr.ID, err)
return "", err
}
// find the image to commit (ctr.Config.Image is the repotag)
image, err := docker.FindImage(ctr.Config.Image, false)
if err != nil {
glog.Errorf("Could not find image %s from %s: %s", ctr.Config.Image, dockerID, err)
return "", err
}
// verify the container is not stale (ctr.Image is the UUID)
if !image.ID.IsLatest() || image.UUID != ctr.Image {
err := fmt.Errorf("cannot commit a stale container")
glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
return "", err
}
// verify the tenantID
tenantID, err := dfs.facade.GetTenantID(datastore.Get(), image.ID.User)
if err != nil {
glog.Errorf("Could not look up tenant %s from image %s for container %s: %s", image.ID.User, image.ID, dockerID, err)
return "", err
} else if tenantID != image.ID.User {
err := fmt.Errorf("service is not the tenant")
glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
return "", err
}
// check the number of image layers
if layers, err := image.History(); err != nil {
glog.Errorf("Could not check history for image %s: %s", image.ID, err)
return "", err
} else if numLayers := len(layers); numLayers >= layer.WARN_LAYER_COUNT {
glog.Warningf("Image %s has %d layers and is approaching the maximum (%d). Please squash image layers.",
image.ID, numLayers, layer.MAX_LAYER_COUNT)
} else {
glog.V(3).Infof("Image %s has %d layers", image.ID, numLayers)
}
// commit the container to the image and tag
newImage, err := ctr.Commit(image.ID.BaseName())
if err != nil {
glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
return "", err
}
// desynchronize any running containers
if err := dfs.desynchronize(newImage); err != nil {
glog.Warningf("Could not denote all desynchronized services: %s", err)
}
// snapshot the filesystem and images
snapshotID, err := dfs.Snapshot(tenantID)
if err != nil {
glog.Errorf("Could not create a snapshot of the new image %s: %s", tenantID, err)
return "", err
}
return snapshotID, nil
}
