func restoreApp(bkup *appCfg, env string) error {
fmt.Println("restoring", bkup.Name)
var svcCfg gconfig.App
exists, err := configStore.AppExists(bkup.Name, env)
if err != nil {
return err
}
if exists {
svcCfg, err = configStore.GetApp(bkup.Name, env)
if err != nil {
return err
}
}
if svcCfg == nil {
svcCfg = configStore.NewAppConfig(bkup.Name, bkup.Version)
}
for k, v := range bkup.Env {
svcCfg.EnvSet(k, v)
}
_, err = configStore.UpdateApp(svcCfg, env)
return err
}
func pullImage(appCfg config.App) (*docker.Image, error) {
image, err := serviceRuntime.PullImage(appCfg.Version(), appCfg.VersionID())
if image == nil || err != nil {
log.Errorf("ERROR: Could not pull image %s: %s", appCfg.Version(), err)
return nil, err
}
log.Printf("Pulled %s version %s\n", appCfg.Name(), appCfg.Version())
return image, nil
}
func (s *ServiceRuntime) Stop(appCfg config.App) error {
containers, err := s.ManagedContainers()
if err != nil {
return err
}
for _, container := range containers {
cenv := s.EnvFor(container)
if cenv["GALAXY_APP"] == appCfg.Name() && cenv["GALAXY_VERSION"] == strconv.FormatInt(appCfg.ID(), 10) {
return s.stopContainer(container)
}
}
return nil
}
// inspectImage checks that the running image matches the config.
// We only use this to print warnings, since we likely need to deploy a new
// config version to fix the inconsistency.
func inspectImage(appCfg config.App) {
image, err := serviceRuntime.InspectImage(appCfg.Version())
if err != nil {
log.Println("error inspecting image", appCfg.Version())
return
}
if utils.StripSHA(image.ID) != appCfg.VersionID() {
log.Printf("warning: %s image ID does not match config", appCfg.Name())
}
}
func (s *ServiceRuntime) StopOldVersion(appCfg config.App, limit int) error {
containers, err := s.ManagedContainers()
if err != nil {
return err
}
stopped := 0
for _, container := range containers {
if stopped == limit {
return nil
}
env := s.EnvFor(container)
// Container name does match one that would be started w/ this service config
if env["GALAXY_APP"] != appCfg.Name() {
continue
}
image, err := s.InspectImage(container.Image)
if err != nil {
log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
continue
}
if image == nil {
log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
continue
}
version := env["GALAXY_VERSION"]
imageDiffers := image.ID != appCfg.VersionID() && appCfg.VersionID() != ""
versionDiffers := version != strconv.FormatInt(appCfg.ID(), 10) && version != ""
if imageDiffers || versionDiffers {
s.stopContainer(container)
stopped = stopped + 1
}
}
return nil
}
func (s *ServiceRuntime) Start(env, pool string, appCfg config.App) (*docker.Container, error) {
img := appCfg.Version()
imgIdRef := appCfg.Version()
if appCfg.VersionID() != "" {
imgIdRef = appCfg.VersionID()
}
// see if we have the image locally
image, err := s.PullImage(img, imgIdRef)
if err != nil {
return nil, err
}
// setup env vars from etcd
var envVars []string
envVars = append(envVars, "ENV"+"="+env)
for key, value := range appCfg.Env() {
if key == "ENV" {
continue
}
envVars = append(envVars, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
}
instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
return nil, err
}
envVars = append(envVars, fmt.Sprintf("HOST_IP=%s", s.hostIP))
envVars = append(envVars, fmt.Sprintf("GALAXY_APP=%s", appCfg.Name()))
envVars = append(envVars, fmt.Sprintf("GALAXY_VERSION=%s", strconv.FormatInt(appCfg.ID(), 10)))
envVars = append(envVars, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))
publicDns, err := EC2PublicHostname()
if err != nil {
log.Warnf("Unable to determine public hostname. Not on AWS? %s", err)
publicDns = "127.0.0.1"
}
envVars = append(envVars, fmt.Sprintf("PUBLIC_HOSTNAME=%s", publicDns))
containerName := appCfg.ContainerName() + "." + strconv.FormatInt(int64(instanceId), 10)
container, err := s.dockerClient.InspectContainer(containerName)
_, ok := err.(*docker.NoSuchContainer)
if err != nil && !ok {
return nil, err
}
// Existing container is running or stopped. If the image has changed, stop
// and re-create it.
if container != nil && container.Image != image.ID {
if container.State.Running || container.State.Restarting || container.State.Paused {
log.Printf("Stopping %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
err := s.dockerClient.StopContainer(container.ID, 10)
if err != nil {
return nil, err
}
}
log.Printf("Removing %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
err = s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
ID: container.ID,
})
if err != nil {
return nil, err
}
container = nil
}
if container == nil {
config := &docker.Config{
Image: img,
Env: envVars,
}
mem := appCfg.GetMemory(pool)
if mem != "" {
m, err := utils.ParseMemory(mem)
if err != nil {
return nil, err
}
config.Memory = m
}
cpu := appCfg.GetCPUShares(pool)
if cpu != "" {
if c, err := strconv.Atoi(cpu); err == nil {
config.CPUShares = int64(c)
}
}
log.Printf("Creating %s version %s", appCfg.Name(), appCfg.Version())
container, err = s.dockerClient.CreateContainer(docker.CreateContainerOptions{
Name: containerName,
Config: config,
})
if err != nil {
return nil, err
}
}
log.Printf("Starting %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
config := &docker.HostConfig{
PublishAllPorts: true,
RestartPolicy: docker.RestartPolicy{
Name: "on-failure",
MaximumRetryCount: 16,
},
}
if s.dns != "" {
config.DNS = []string{s.dns}
}
err = s.dockerClient.StartContainer(container.ID, config)
return container, err
}
func (s *ServiceRuntime) StartInteractive(env, pool string, appCfg config.App) error {
// see if we have the image locally
fmt.Fprintf(os.Stderr, "Pulling latest image for %s\n", appCfg.Version())
_, err := s.PullImage(appCfg.Version(), appCfg.VersionID())
if err != nil {
return err
}
args := []string{
"run", "--rm", "-i",
}
args = append(args, "-e")
args = append(args, "ENV"+"="+env)
for key, value := range appCfg.Env() {
if key == "ENV" {
continue
}
args = append(args, "-e")
args = append(args, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
}
args = append(args, "-e")
args = append(args, fmt.Sprintf("HOST_IP=%s", s.hostIP))
if s.dns != "" {
args = append(args, "--dns")
args = append(args, s.dns)
}
args = append(args, "-e")
args = append(args, fmt.Sprintf("GALAXY_APP=%s", appCfg.Name()))
args = append(args, "-e")
args = append(args, fmt.Sprintf("GALAXY_VERSION=%s", strconv.FormatInt(appCfg.ID(), 10)))
instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
return err
}
args = append(args, "-e")
args = append(args, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))
publicDns, err := EC2PublicHostname()
if err != nil {
log.Warnf("Unable to determine public hostname. Not on AWS? %s", err)
publicDns = "127.0.0.1"
}
args = append(args, "-e")
args = append(args, fmt.Sprintf("PUBLIC_HOSTNAME=%s", publicDns))
mem := appCfg.GetMemory(pool)
if mem != "" {
args = append(args, "-m")
args = append(args, mem)
}
cpu := appCfg.GetCPUShares(pool)
if cpu != "" {
args = append(args, "-c")
args = append(args, cpu)
}
args = append(args, []string{"-t", appCfg.Version(), "/bin/bash"}...)
// shell out to docker run to get signal forwarded and terminal setup correctly
//cmd := exec.Command("docker", "run", "-rm", "-i", "-t", appCfg.Version(), "/bin/bash")
cmd := exec.Command("docker", args...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Start()
if err != nil {
log.Fatal(err)
}
err = cmd.Wait()
if err != nil {
fmt.Fprintf(os.Stderr, "Command finished with error: %v\n", err)
}
return err
}
func (s *ServiceRuntime) RunCommand(env string, appCfg config.App, cmd []string) (*docker.Container, error) {
// see if we have the image locally
fmt.Fprintf(os.Stderr, "Pulling latest image for %s\n", appCfg.Version())
_, err := s.PullImage(appCfg.Version(), appCfg.VersionID())
if err != nil {
return nil, err
}
instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
return nil, err
}
envVars := []string{"ENV=" + env}
for key, value := range appCfg.Env() {
if key == "ENV" {
continue
}
envVars = append(envVars, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
}
envVars = append(envVars, "GALAXY_APP="+appCfg.Name())
envVars = append(envVars, "GALAXY_VERSION="+strconv.FormatInt(appCfg.ID(), 10))
envVars = append(envVars, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))
runCmd := []string{"/bin/bash", "-c", strings.Join(cmd, " ")}
container, err := s.dockerClient.CreateContainer(docker.CreateContainerOptions{
Config: &docker.Config{
Image: appCfg.Version(),
Env: envVars,
AttachStdout: true,
AttachStderr: true,
Cmd: runCmd,
OpenStdin: false,
},
})
if err != nil {
return nil, err
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill)
go func(s *ServiceRuntime, containerId string) {
<-c
log.Println("Stopping container...")
err := s.dockerClient.StopContainer(containerId, 3)
if err != nil {
log.Printf("ERROR: Unable to stop container: %s", err)
}
err = s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
ID: containerId,
})
if err != nil {
log.Printf("ERROR: Unable to stop container: %s", err)
}
}(s, container.ID)
defer s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
ID: container.ID,
})
config := &docker.HostConfig{}
if s.dns != "" {
config.DNS = []string{s.dns}
}
err = s.dockerClient.StartContainer(container.ID, config)
if err != nil {
return container, err
}
// FIXME: Hack to work around the race of attaching to a container before it's
// actually running. Tried polling the container and then attaching but the
// output gets lost sometimes if the command executes very quickly. Not sure
// what's going on.
time.Sleep(1 * time.Second)
err = s.dockerClient.AttachToContainer(docker.AttachToContainerOptions{
Container: container.ID,
OutputStream: os.Stdout,
ErrorStream: os.Stderr,
Logs: true,
Stream: false,
Stdout: true,
Stderr: true,
})
if err != nil {
log.Printf("ERROR: Unable to attach to running container: %s", err.Error())
}
s.dockerClient.WaitContainer(container.ID)
return container, err
}
func startService(appCfg config.App, logStatus bool) {
desired, err := commander.Balanced(configStore, hostIP, appCfg.Name(), env, pool)
if err != nil {
log.Errorf("ERROR: Could not determine instance count: %s", err)
return
}
running, err := serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
log.Errorf("ERROR: Could not determine running instance count: %s", err)
return
}
for i := 0; i < desired-running; i++ {
container, err := serviceRuntime.Start(env, pool, appCfg)
if err != nil {
log.Errorf("ERROR: Could not start containers: %s", err)
return
}
log.Printf("Started %s version %s as %s\n", appCfg.Name(), appCfg.Version(), container.ID[0:12])
err = serviceRuntime.StopOldVersion(appCfg, 1)
if err != nil {
log.Errorf("ERROR: Could not stop containers: %s", err)
}
}
running, err = serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
log.Errorf("ERROR: Could not determine running instance count: %s", err)
return
}
for i := 0; i < running-desired; i++ {
err := serviceRuntime.Stop(appCfg)
if err != nil {
log.Errorf("ERROR: Could not stop container: %s", err)
}
}
err = serviceRuntime.StopOldVersion(appCfg, -1)
if err != nil {
log.Errorf("ERROR: Could not stop old containers: %s", err)
}
// check the image version, and log any inconsistencies
inspectImage(appCfg)
}
func pullImage(appCfg config.App) (*docker.Image, error) {
image, err := serviceRuntime.InspectImage(appCfg.Version())
if image != nil && image.ID == appCfg.VersionID() || appCfg.VersionID() == "" {
return image, nil
}
log.Printf("Pulling %s version %s\n", appCfg.Name(), appCfg.Version())
image, err = serviceRuntime.PullImage(appCfg.Version(),
appCfg.VersionID())
if image == nil || err != nil {
log.Errorf("ERROR: Could not pull image %s: %s",
appCfg.Version(), err)
return nil, err
}
if image.ID != appCfg.VersionID() && len(appCfg.VersionID()) > 12 {
log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
appCfg.Version(),
image.ID[0:12], appCfg.VersionID()[0:12])
return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", appCfg.VersionID()[0:12]))
}
log.Printf("Pulled %s\n", appCfg.Version())
return image, nil
}