Ejemplo n.º 1
0
func restoreApp(bkup *appCfg, env string) error {
	fmt.Println("restoring", bkup.Name)

	var svcCfg gconfig.App

	exists, err := configStore.AppExists(bkup.Name, env)
	if err != nil {
		return err
	}

	if exists {
		svcCfg, err = configStore.GetApp(bkup.Name, env)
		if err != nil {
			return err
		}
	}

	if svcCfg == nil {
		svcCfg = configStore.NewAppConfig(bkup.Name, bkup.Version)
	}

	for k, v := range bkup.Env {
		svcCfg.EnvSet(k, v)
	}

	_, err = configStore.UpdateApp(svcCfg, env)
	return err
}
Ejemplo n.º 2
0
func pullImage(appCfg config.App) (*docker.Image, error) {
	image, err := serviceRuntime.PullImage(appCfg.Version(), appCfg.VersionID())
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s", appCfg.Version(), err)
		return nil, err
	}

	log.Printf("Pulled %s version %s\n", appCfg.Name(), appCfg.Version())
	return image, nil
}
Ejemplo n.º 3
0
func (s *ServiceRuntime) Stop(appCfg config.App) error {
	containers, err := s.ManagedContainers()
	if err != nil {
		return err
	}

	for _, container := range containers {
		cenv := s.EnvFor(container)
		if cenv["GALAXY_APP"] == appCfg.Name() && cenv["GALAXY_VERSION"] == strconv.FormatInt(appCfg.ID(), 10) {
			return s.stopContainer(container)
		}
	}
	return nil
}
Ejemplo n.º 4
0
// inspectImage checks that the running image matches the config.
// We only use this to print warnings, since we likely need to deploy a new
// config version to fix the inconsistency.
func inspectImage(appCfg config.App) {
	image, err := serviceRuntime.InspectImage(appCfg.Version())
	if err != nil {
		log.Println("error inspecting image", appCfg.Version())
		return
	}

	if utils.StripSHA(image.ID) != appCfg.VersionID() {
		log.Printf("warning: %s image ID does not match config", appCfg.Name())
	}
}
Ejemplo n.º 5
0
func (s *ServiceRuntime) StopOldVersion(appCfg config.App, limit int) error {
	containers, err := s.ManagedContainers()
	if err != nil {
		return err
	}

	stopped := 0

	for _, container := range containers {

		if stopped == limit {
			return nil
		}

		env := s.EnvFor(container)
		// Container name does match one that would be started w/ this service config
		if env["GALAXY_APP"] != appCfg.Name() {
			continue
		}

		image, err := s.InspectImage(container.Image)
		if err != nil {
			log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
			continue
		}

		if image == nil {
			log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
			continue

		}

		version := env["GALAXY_VERSION"]

		imageDiffers := image.ID != appCfg.VersionID() && appCfg.VersionID() != ""
		versionDiffers := version != strconv.FormatInt(appCfg.ID(), 10) && version != ""

		if imageDiffers || versionDiffers {
			s.stopContainer(container)
			stopped = stopped + 1
		}
	}
	return nil
}
Ejemplo n.º 6
0
func (s *ServiceRuntime) Start(env, pool string, appCfg config.App) (*docker.Container, error) {

	img := appCfg.Version()

	imgIdRef := appCfg.Version()
	if appCfg.VersionID() != "" {
		imgIdRef = appCfg.VersionID()
	}
	// see if we have the image locally
	image, err := s.PullImage(img, imgIdRef)
	if err != nil {
		return nil, err
	}

	// setup env vars from etcd
	var envVars []string
	envVars = append(envVars, "ENV"+"="+env)

	for key, value := range appCfg.Env() {
		if key == "ENV" {
			continue
		}
		envVars = append(envVars, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
	}

	instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		return nil, err
	}

	envVars = append(envVars, fmt.Sprintf("HOST_IP=%s", s.hostIP))
	envVars = append(envVars, fmt.Sprintf("GALAXY_APP=%s", appCfg.Name()))
	envVars = append(envVars, fmt.Sprintf("GALAXY_VERSION=%s", strconv.FormatInt(appCfg.ID(), 10)))
	envVars = append(envVars, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))

	publicDns, err := EC2PublicHostname()
	if err != nil {
		log.Warnf("Unable to determine public hostname. Not on AWS? %s", err)
		publicDns = "127.0.0.1"
	}
	envVars = append(envVars, fmt.Sprintf("PUBLIC_HOSTNAME=%s", publicDns))

	containerName := appCfg.ContainerName() + "." + strconv.FormatInt(int64(instanceId), 10)
	container, err := s.dockerClient.InspectContainer(containerName)
	_, ok := err.(*docker.NoSuchContainer)
	if err != nil && !ok {
		return nil, err
	}

	// Existing container is running or stopped.  If the image has changed, stop
	// and re-create it.
	if container != nil && container.Image != image.ID {
		if container.State.Running || container.State.Restarting || container.State.Paused {
			log.Printf("Stopping %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
			err := s.dockerClient.StopContainer(container.ID, 10)
			if err != nil {
				return nil, err
			}
		}

		log.Printf("Removing %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
		err = s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
			ID: container.ID,
		})
		if err != nil {
			return nil, err
		}
		container = nil
	}

	if container == nil {

		config := &docker.Config{
			Image: img,
			Env:   envVars,
		}

		mem := appCfg.GetMemory(pool)
		if mem != "" {
			m, err := utils.ParseMemory(mem)
			if err != nil {
				return nil, err
			}
			config.Memory = m
		}

		cpu := appCfg.GetCPUShares(pool)
		if cpu != "" {
			if c, err := strconv.Atoi(cpu); err == nil {
				config.CPUShares = int64(c)
			}
		}

		log.Printf("Creating %s version %s", appCfg.Name(), appCfg.Version())
		container, err = s.dockerClient.CreateContainer(docker.CreateContainerOptions{
			Name:   containerName,
			Config: config,
		})
		if err != nil {
			return nil, err
		}
	}

	log.Printf("Starting %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])

	config := &docker.HostConfig{
		PublishAllPorts: true,
		RestartPolicy: docker.RestartPolicy{
			Name:              "on-failure",
			MaximumRetryCount: 16,
		},
	}

	if s.dns != "" {
		config.DNS = []string{s.dns}
	}
	err = s.dockerClient.StartContainer(container.ID, config)

	return container, err
}
Ejemplo n.º 7
0
func (s *ServiceRuntime) StartInteractive(env, pool string, appCfg config.App) error {

	// see if we have the image locally
	fmt.Fprintf(os.Stderr, "Pulling latest image for %s\n", appCfg.Version())
	_, err := s.PullImage(appCfg.Version(), appCfg.VersionID())
	if err != nil {
		return err
	}

	args := []string{
		"run", "--rm", "-i",
	}
	args = append(args, "-e")
	args = append(args, "ENV"+"="+env)

	for key, value := range appCfg.Env() {
		if key == "ENV" {
			continue
		}

		args = append(args, "-e")
		args = append(args, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
	}

	args = append(args, "-e")
	args = append(args, fmt.Sprintf("HOST_IP=%s", s.hostIP))
	if s.dns != "" {
		args = append(args, "--dns")
		args = append(args, s.dns)
	}
	args = append(args, "-e")
	args = append(args, fmt.Sprintf("GALAXY_APP=%s", appCfg.Name()))
	args = append(args, "-e")
	args = append(args, fmt.Sprintf("GALAXY_VERSION=%s", strconv.FormatInt(appCfg.ID(), 10)))

	instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		return err
	}
	args = append(args, "-e")
	args = append(args, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))

	publicDns, err := EC2PublicHostname()
	if err != nil {
		log.Warnf("Unable to determine public hostname. Not on AWS? %s", err)
		publicDns = "127.0.0.1"
	}

	args = append(args, "-e")
	args = append(args, fmt.Sprintf("PUBLIC_HOSTNAME=%s", publicDns))

	mem := appCfg.GetMemory(pool)
	if mem != "" {
		args = append(args, "-m")
		args = append(args, mem)
	}

	cpu := appCfg.GetCPUShares(pool)
	if cpu != "" {
		args = append(args, "-c")
		args = append(args, cpu)
	}

	args = append(args, []string{"-t", appCfg.Version(), "/bin/bash"}...)
	// shell out to docker run to get signal forwarded and terminal setup correctly
	//cmd := exec.Command("docker", "run", "-rm", "-i", "-t", appCfg.Version(), "/bin/bash")
	cmd := exec.Command("docker", args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Command finished with error: %v\n", err)
	}

	return err
}
Ejemplo n.º 8
0
func (s *ServiceRuntime) RunCommand(env string, appCfg config.App, cmd []string) (*docker.Container, error) {

	// see if we have the image locally
	fmt.Fprintf(os.Stderr, "Pulling latest image for %s\n", appCfg.Version())
	_, err := s.PullImage(appCfg.Version(), appCfg.VersionID())
	if err != nil {
		return nil, err
	}

	instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		return nil, err
	}

	envVars := []string{"ENV=" + env}

	for key, value := range appCfg.Env() {
		if key == "ENV" {
			continue
		}
		envVars = append(envVars, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
	}
	envVars = append(envVars, "GALAXY_APP="+appCfg.Name())
	envVars = append(envVars, "GALAXY_VERSION="+strconv.FormatInt(appCfg.ID(), 10))
	envVars = append(envVars, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))

	runCmd := []string{"/bin/bash", "-c", strings.Join(cmd, " ")}

	container, err := s.dockerClient.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image:        appCfg.Version(),
			Env:          envVars,
			AttachStdout: true,
			AttachStderr: true,
			Cmd:          runCmd,
			OpenStdin:    false,
		},
	})

	if err != nil {
		return nil, err
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	go func(s *ServiceRuntime, containerId string) {
		<-c
		log.Println("Stopping container...")
		err := s.dockerClient.StopContainer(containerId, 3)
		if err != nil {
			log.Printf("ERROR: Unable to stop container: %s", err)
		}
		err = s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
			ID: containerId,
		})
		if err != nil {
			log.Printf("ERROR: Unable to stop container: %s", err)
		}

	}(s, container.ID)

	defer s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})
	config := &docker.HostConfig{}
	if s.dns != "" {
		config.DNS = []string{s.dns}
	}
	err = s.dockerClient.StartContainer(container.ID, config)

	if err != nil {
		return container, err
	}

	// FIXME: Hack to work around the race of attaching to a container before it's
	// actually running.  Tried polling the container and then attaching but the
	// output gets lost sometimes if the command executes very quickly. Not sure
	// what's going on.
	time.Sleep(1 * time.Second)

	err = s.dockerClient.AttachToContainer(docker.AttachToContainerOptions{
		Container:    container.ID,
		OutputStream: os.Stdout,
		ErrorStream:  os.Stderr,
		Logs:         true,
		Stream:       false,
		Stdout:       true,
		Stderr:       true,
	})

	if err != nil {
		log.Printf("ERROR: Unable to attach to running container: %s", err.Error())
	}

	s.dockerClient.WaitContainer(container.ID)

	return container, err
}
Ejemplo n.º 9
0
func startService(appCfg config.App, logStatus bool) {

	desired, err := commander.Balanced(configStore, hostIP, appCfg.Name(), env, pool)
	if err != nil {
		log.Errorf("ERROR: Could not determine instance count: %s", err)
		return
	}

	running, err := serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		log.Errorf("ERROR: Could not determine running instance count: %s", err)
		return
	}

	for i := 0; i < desired-running; i++ {
		container, err := serviceRuntime.Start(env, pool, appCfg)
		if err != nil {
			log.Errorf("ERROR: Could not start containers: %s", err)
			return
		}

		log.Printf("Started %s version %s as %s\n", appCfg.Name(), appCfg.Version(), container.ID[0:12])

		err = serviceRuntime.StopOldVersion(appCfg, 1)
		if err != nil {
			log.Errorf("ERROR: Could not stop containers: %s", err)
		}
	}

	running, err = serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		log.Errorf("ERROR: Could not determine running instance count: %s", err)
		return
	}

	for i := 0; i < running-desired; i++ {
		err := serviceRuntime.Stop(appCfg)
		if err != nil {
			log.Errorf("ERROR: Could not stop container: %s", err)
		}
	}

	err = serviceRuntime.StopOldVersion(appCfg, -1)
	if err != nil {
		log.Errorf("ERROR: Could not stop old containers: %s", err)
	}

	// check the image version, and log any inconsistencies
	inspectImage(appCfg)
}
Ejemplo n.º 10
0
func pullImage(appCfg config.App) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(appCfg.Version())
	if image != nil && image.ID == appCfg.VersionID() || appCfg.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", appCfg.Name(), appCfg.Version())
	image, err = serviceRuntime.PullImage(appCfg.Version(),
		appCfg.VersionID())
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			appCfg.Version(), err)
		return nil, err
	}

	if image.ID != appCfg.VersionID() && len(appCfg.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			appCfg.Version(),
			image.ID[0:12], appCfg.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", appCfg.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", appCfg.Version())
	return image, nil
}