Example #1
0
func (svc *IService) remove(notify chan<- int) {
	defer close(notify)
	ctr, err := docker.FindContainer(svc.name())
	if err == docker.ErrNoSuchContainer {
		return
	} else if err != nil {
		glog.Errorf("Could not get isvc container %s", svc.Name)
		return
	}

	// report the log output
	if output, err := exec.Command("docker", "logs", "--tail", "1000", ctr.ID).CombinedOutput(); err != nil {
		glog.Warningf("Could not get logs for container %s", ctr.Name)
	} else {
		glog.V(1).Infof("Exited isvc %s:\n %s", svc.Name, string(output))
	}

	// kill the container if it is running
	if ctr.IsRunning() {
		glog.Warningf("isvc %s is still running; killing", svc.Name)
		ctr.Kill()
	}

	// get the exit code
	rc, _ := ctr.Wait(time.Second)
	defer func() { notify <- rc }()

	// delete the container
	if err := ctr.Delete(true); err != nil {
		glog.Errorf("Could not remove isvc %s: %s", ctr.Name, err)
	}
}
Example #2
0
// StopService terminates a particular service instance (serviceState) on the localhost.
func (a *HostAgent) StopService(state *servicestate.ServiceState) error {
	if state == nil || state.DockerID == "" {
		return errors.New("missing Docker ID")
	}

	ctr, err := docker.FindContainer(state.DockerID)
	if err != nil {
		return err
	}

	return ctr.Stop(45 * time.Second)
}
Example #3
0
func (svc *IService) Exec(command []string) error {
	ctr, err := docker.FindContainer(svc.name())
	if err != nil {
		return err
	}

	output, err := utils.AttachAndRun(ctr.ID, command)
	if err != nil {
		return err
	}
	os.Stdout.Write(output)
	return nil
}
Example #4
0
func (svc *IService) stop() error {
	ctr, err := docker.FindContainer(svc.name())
	if err == docker.ErrNoSuchContainer {
		svc.setStoppedHealthStatus(nil)
		return nil
	} else if err != nil {
		glog.Errorf("Could not get isvc container %s", svc.Name)
		svc.setStoppedHealthStatus(err)
		return err
	}

	glog.Warningf("Stopping isvc container %s", svc.name())
	err = ctr.Stop(45 * time.Second)
	svc.setStoppedHealthStatus(err)
	return err
}
Example #5
0
func (svc *IService) attach() (*docker.Container, error) {
	ctr, _ := docker.FindContainer(svc.name())
	if ctr != nil {
		notify := make(chan int, 1)
		if !ctr.IsRunning() {
			glog.Infof("isvc %s found but not running; removing container %s", svc.name(), ctr.ID)
			go svc.remove(notify)
		} else if !svc.checkvolumes(ctr) {
			glog.Infof("isvc %s found but volumes are missing or incomplete; removing container %s", svc.name(), ctr.ID)
			ctr.OnEvent(docker.Die, func(cid string) { svc.remove(notify) })
			svc.stop()
		} else {
			glog.Infof("Attaching to isvc %s at %s", svc.name(), ctr.ID)
			return ctr, nil
		}
		<-notify
	}

	glog.Infof("Creating a new container for isvc %s", svc.name())
	return svc.create()
}
Example #6
0
// AttachService attempts to attach to a running container
func (a *HostAgent) AttachService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
	ctr, err := docker.FindContainer(state.DockerID)
	if err != nil {
		return err
	}

	if !ctr.IsRunning() {
		defer exited(state.ID)
		return nil
	}

	ctr.OnEvent(docker.Die, func(cid string) {
		defer exited(state.ID)
		glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
		state.DockerID = cid
		a.removeInstance(state.ID, ctr)
	})

	go a.setProxy(svc, ctr)
	return nil
}
Example #7
0
func (svc *IService) stats(halt <-chan struct{}) {
	registry := metrics.NewRegistry()
	tc := time.Tick(10 * time.Second)

	for {
		select {
		case <-halt:
			glog.Infof("stop collecting stats for %s", svc.Name)
			return
		case t := <-tc:
			ctr, err := docker.FindContainer(svc.name())
			if err != nil {
				glog.Warningf("Could not find container for isvc %s: %s", svc.Name, err)
				break
			}

			if cpuacctStat, err := cgroup.ReadCpuacctStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Cpuacct)); err != nil {
				glog.Warningf("Could not read CpuacctStat for isvc %s: %s", svc.Name, err)
				break
			} else {
				metrics.GetOrRegisterGauge("CpuacctStat.system", registry).Update(cpuacctStat.System)
				metrics.GetOrRegisterGauge("CpuacctStat.user", registry).Update(cpuacctStat.User)
			}

			if memoryStat, err := cgroup.ReadMemoryStat(cgroup.GetCgroupDockerStatsFilePath(ctr.ID, cgroup.Memory)); err != nil {
				glog.Warningf("Could not read MemoryStat for isvc %s: %s", svc.Name, err)
				break
			} else {
				metrics.GetOrRegisterGauge("cgroup.memory.pgmajfault", registry).Update(memoryStat.Pgfault)
				metrics.GetOrRegisterGauge("cgroup.memory.totalrss", registry).Update(memoryStat.TotalRss)
				metrics.GetOrRegisterGauge("cgroup.memory.cache", registry).Update(memoryStat.Cache)
			}
			// Gather the stats
			stats := []containerStat{}
			registry.Each(func(name string, i interface{}) {
				if metric, ok := i.(metrics.Gauge); ok {
					tagmap := make(map[string]string)
					tagmap["isvcname"] = svc.Name
					stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
				}
				if metricf64, ok := i.(metrics.GaugeFloat64); ok {
					tagmap := make(map[string]string)
					tagmap["isvcname"] = svc.Name
					stats = append(stats, containerStat{name, strconv.FormatFloat(metricf64.Value(), 'f', -1, 32), t.Unix(), tagmap})
				}
			})
			// Post the stats.
			data, err := json.Marshal(stats)
			if err != nil {
				glog.Warningf("Error marshalling isvc stats json.")
				break
			}

			req, err := http.NewRequest("POST", "http://127.0.0.1:4242/api/put", bytes.NewBuffer(data))
			if err != nil {
				glog.Warningf("Error creating isvc stats request.")
				break
			}
			resp, err := http.DefaultClient.Do(req)
			if err != nil {
				glog.V(4).Infof("Error making isvc stats request.")
				break
			}
			if strings.Contains(resp.Status, "204 No Content") == false {
				glog.Warningf("Couldn't post stats:", resp.Status)
				break
			}
		}
	}
}
Example #8
0
// StartService starts a new instance of the specified service and updates the control center state accordingly.
func (a *HostAgent) StartService(svc *service.Service, state *servicestate.ServiceState, exited func(string)) error {
	glog.V(2).Infof("About to start service %s with name %s", svc.ID, svc.Name)
	client, err := NewControlClient(a.master)
	if err != nil {
		glog.Errorf("Could not start ControlPlane client %v", err)
		return err
	}
	defer client.Close()

	// start from a known good state
	if state.DockerID != "" {
		if ctr, err := docker.FindContainer(state.DockerID); err != nil {
			glog.Errorf("Could not find container %s for %s", state.DockerID, state.ID)
		} else if err := ctr.Delete(true); err != nil {
			glog.Errorf("Could not delete container %s for %s", state.DockerID, state.ID)
		}
	}

	// create the docker client Config and HostConfig structures necessary to create and start the service
	config, hostconfig, err := configureContainer(a, client, svc, state, a.virtualAddressSubnet)
	if err != nil {
		glog.Errorf("can't configure container: %v", err)
		return err
	}

	cjson, _ := json.MarshalIndent(config, "", "     ")
	glog.V(3).Infof(">>> CreateContainerOptions:\n%s", string(cjson))

	hcjson, _ := json.MarshalIndent(hostconfig, "", "     ")
	glog.V(3).Infof(">>> HostConfigOptions:\n%s", string(hcjson))

	cd := &docker.ContainerDefinition{
		dockerclient.CreateContainerOptions{Name: state.ID, Config: config},
		*hostconfig,
	}

	ctr, err := docker.NewContainer(cd, false, 10*time.Second, nil, nil)
	if err != nil {
		glog.Errorf("Error trying to create container %v: %v", config, err)
		return err
	}

	var started sync.WaitGroup
	started.Add(1)
	ctr.OnEvent(docker.Start, func(cid string) {
		glog.Infof("Instance %s (%s) for %s (%s) has started", state.ID, ctr.ID, svc.Name, svc.ID)
		started.Done()
	})

	ctr.OnEvent(docker.Die, func(cid string) {
		defer exited(state.ID)
		glog.Infof("Instance %s (%s) for %s (%s) has died", state.ID, ctr.ID, svc.Name, svc.ID)
		state.DockerID = cid
		a.removeInstance(state.ID, ctr)
	})

	if err := ctr.Start(time.Hour); err != nil {
		glog.Errorf("Could not start service state %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
		a.removeInstance(state.ID, ctr)
		return err
	}

	started.Wait()
	if err := updateInstance(state, ctr); err != nil {
		glog.Errorf("Could not update instance %s (%s) for service %s (%s): %s", state.ID, ctr.ID, svc.Name, svc.ID, err)
		ctr.Stop(45 * time.Second)
		return err
	}

	go a.setProxy(svc, ctr)
	return nil
}
Example #9
0
// Get the state of the docker container given the dockerId
func getDockerState(dockerID string) (*docker.Container, error) {
	glog.V(1).Infof("Inspecting container: %s", dockerID)
	return docker.FindContainer(dockerID)
}
Example #10
0
// Commit will merge a container into existing services' image
func (dfs *DistributedFilesystem) Commit(dockerID string) (string, error) {
	// get the container
	ctr, err := docker.FindContainer(dockerID)
	if err != nil {
		glog.Errorf("Could not get container %s: %s", dockerID, err)
		return "", err
	}

	// verify the container is not currently running
	if ctr.IsRunning() {
		err := fmt.Errorf("cannot commit a running container")
		glog.Errorf("Error committing container %s: %s", ctr.ID, err)
		return "", err
	}

	// find the image to commit (ctr.Config.Image is the repotag)
	image, err := docker.FindImage(ctr.Config.Image, false)
	if err != nil {
		glog.Errorf("Could not find image %s from %s: %s", ctr.Config.Image, dockerID, err)
		return "", err
	}

	// verify the container is not stale (ctr.Image is the UUID)
	if !image.ID.IsLatest() || image.UUID != ctr.Image {
		err := fmt.Errorf("cannot commit a stale container")
		glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
		return "", err
	}

	// verify the tenantID
	tenantID, err := dfs.facade.GetTenantID(datastore.Get(), image.ID.User)
	if err != nil {
		glog.Errorf("Could not look up tenant %s from image %s for container %s: %s", image.ID.User, image.ID, dockerID, err)
		return "", err
	} else if tenantID != image.ID.User {
		err := fmt.Errorf("service is not the tenant")
		glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
		return "", err
	}

	// check the number of image layers
	if layers, err := image.History(); err != nil {
		glog.Errorf("Could not check history for image %s: %s", image.ID, err)
		return "", err
	} else if numLayers := len(layers); numLayers >= layer.WARN_LAYER_COUNT {
		glog.Warningf("Image %s has %d layers and is approaching the maximum (%d). Please squash image layers.",
			image.ID, numLayers, layer.MAX_LAYER_COUNT)
	} else {
		glog.V(3).Infof("Image %s has %d layers", image.ID, numLayers)
	}

	// commit the container to the image and tag
	newImage, err := ctr.Commit(image.ID.BaseName())
	if err != nil {
		glog.Errorf("Could not commit %s (%s): %s", dockerID, image.ID, err)
		return "", err
	}

	// desynchronize any running containers
	if err := dfs.desynchronize(newImage); err != nil {
		glog.Warningf("Could not denote all desynchronized services: %s", err)
	}

	// snapshot the filesystem and images
	snapshotID, err := dfs.Snapshot(tenantID)
	if err != nil {
		glog.Errorf("Could not create a snapshot of the new image %s: %s", tenantID, err)
		return "", err
	}

	return snapshotID, nil
}