Beispiel #1
0
func (s *ServiceRuntime) Start(env, pool string, appCfg config.App) (*docker.Container, error) {

	img := appCfg.Version()

	imgIdRef := appCfg.Version()
	if appCfg.VersionID() != "" {
		imgIdRef = appCfg.VersionID()
	}
	// see if we have the image locally
	image, err := s.PullImage(img, imgIdRef)
	if err != nil {
		return nil, err
	}

	// setup env vars from etcd
	var envVars []string
	envVars = append(envVars, "ENV"+"="+env)

	for key, value := range appCfg.Env() {
		if key == "ENV" {
			continue
		}
		envVars = append(envVars, strings.ToUpper(key)+"="+s.replaceVarEnv(value, s.hostIP))
	}

	instanceId, err := s.NextInstanceSlot(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		return nil, err
	}

	envVars = append(envVars, fmt.Sprintf("HOST_IP=%s", s.hostIP))
	envVars = append(envVars, fmt.Sprintf("GALAXY_APP=%s", appCfg.Name()))
	envVars = append(envVars, fmt.Sprintf("GALAXY_VERSION=%s", strconv.FormatInt(appCfg.ID(), 10)))
	envVars = append(envVars, fmt.Sprintf("GALAXY_INSTANCE=%s", strconv.FormatInt(int64(instanceId), 10)))

	publicDns, err := EC2PublicHostname()
	if err != nil {
		log.Warnf("Unable to determine public hostname. Not on AWS? %s", err)
		publicDns = "127.0.0.1"
	}
	envVars = append(envVars, fmt.Sprintf("PUBLIC_HOSTNAME=%s", publicDns))

	containerName := appCfg.ContainerName() + "." + strconv.FormatInt(int64(instanceId), 10)
	container, err := s.dockerClient.InspectContainer(containerName)
	_, ok := err.(*docker.NoSuchContainer)
	if err != nil && !ok {
		return nil, err
	}

	// Existing container is running or stopped.  If the image has changed, stop
	// and re-create it.
	if container != nil && container.Image != image.ID {
		if container.State.Running || container.State.Restarting || container.State.Paused {
			log.Printf("Stopping %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
			err := s.dockerClient.StopContainer(container.ID, 10)
			if err != nil {
				return nil, err
			}
		}

		log.Printf("Removing %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])
		err = s.dockerClient.RemoveContainer(docker.RemoveContainerOptions{
			ID: container.ID,
		})
		if err != nil {
			return nil, err
		}
		container = nil
	}

	if container == nil {

		config := &docker.Config{
			Image: img,
			Env:   envVars,
		}

		mem := appCfg.GetMemory(pool)
		if mem != "" {
			m, err := utils.ParseMemory(mem)
			if err != nil {
				return nil, err
			}
			config.Memory = m
		}

		cpu := appCfg.GetCPUShares(pool)
		if cpu != "" {
			if c, err := strconv.Atoi(cpu); err == nil {
				config.CPUShares = int64(c)
			}
		}

		log.Printf("Creating %s version %s", appCfg.Name(), appCfg.Version())
		container, err = s.dockerClient.CreateContainer(docker.CreateContainerOptions{
			Name:   containerName,
			Config: config,
		})
		if err != nil {
			return nil, err
		}
	}

	log.Printf("Starting %s version %s running as %s", appCfg.Name(), appCfg.Version(), container.ID[0:12])

	config := &docker.HostConfig{
		PublishAllPorts: true,
		RestartPolicy: docker.RestartPolicy{
			Name:              "on-failure",
			MaximumRetryCount: 16,
		},
	}

	if s.dns != "" {
		config.DNS = []string{s.dns}
	}
	err = s.dockerClient.StartContainer(container.ID, config)

	return container, err
}
Beispiel #2
0
func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&registryURL, "registry", utils.GetEnv("GALAXY_REGISTRY_URL", "redis://127.0.0.1:6379"), "registry URL")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&hostIP, "host-ip", "127.0.0.1", "Host IP")
	flag.StringVar(&shuttleAddr, "shuttle-addr", "", "Shuttle API addr (127.0.0.1:9090)")
	flag.StringVar(&dns, "dns", "", "DNS addr to use for containers")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Usage = func() {
		println("Usage: commander [options] <command> [<args>]\n")
		println("Available commands are:")
		println("   agent           Runs commander agent")
		println("   app             List all apps")
		println("   app:assign      Assign an app to a pool")
		println("   app:create      Create an app")
		println("   app:deploy      Deploy an app")
		println("   app:delete      Delete an app")
		println("   app:restart     Restart an app")
		println("   app:run         Run a command within an app on this host")
		println("   app:shell       Run a shell within an app on this host")
		println("   app:start       Starts one or more apps")
		println("   app:stop        Stops one or more apps")
		println("   app:unassign    Unassign an app from a pool")
		println("   config          List config for an app")
		println("   config:get      Get config values for an app")
		println("   config:set      Set config values for an app")
		println("   config:unset    Unset config values for an app")
		println("   runtime         List container runtime policies")
		println("   runtime:set     Set container runtime policies")
		println("   hosts           List hosts in an env and pool")
		println("\nOptions:\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	log.DefaultLogger = log.New(os.Stdout, "", log.INFO)
	log.DefaultLogger.SetFlags(0)

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.Usage()
		os.Exit(1)
	}

	initOrDie()

	switch flag.Args()[0] {
	case "dump":
		if flag.NArg() < 2 {
			fmt.Println("Usage: commander dump ENV")
			os.Exit(1)
		}
		dump(flag.Arg(1))
		return

	case "restore":
		if flag.NArg() < 2 {
			fmt.Println("Usage: commander dump ENV FILE")
			os.Exit(1)
		}
		restore(flag.Arg(1))
		return

	case "agent":
		log.DefaultLogger.SetFlags(golog.LstdFlags)
		loop = true
		agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
		agentFs.Usage = func() {
			println("Usage: commander agent [options]\n")
			println("    Runs commander continuously\n\n")
			println("Options:\n\n")
			agentFs.PrintDefaults()
		}
		agentFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

	case "app":
		appFs := flag.NewFlagSet("app", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app\n")
			println("    List all apps or apps in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])
		err := commander.AppList(configStore, env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:assign":
		appFs := flag.NewFlagSet("app:assign", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:assign <app>\n")
			println("    Assign an app to a pool\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppAssign(configStore, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "app:create":
		appFs := flag.NewFlagSet("app:create", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:create <app>\n")
			println("    Create an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppCreate(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:delete":
		appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:delete <app>\n")
			println("    Delete an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppDelete(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:deploy":
		appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:deploy [-force] <app> <version>\n")
			println("    Deploy an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() != 2 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppDeploy(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:restart":
		appFs := flag.NewFlagSet("app:restart", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:restart <app>\n")
			println("    Restart an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppRestart(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:run":
		appFs := flag.NewFlagSet("app:run", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:run <app> <cmd>\n")
			println("    Restart an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() < 2 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppRun(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:shell":
		appFs := flag.NewFlagSet("app:shell", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:shell <app>\n")
			println("    Run a shell for an app\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppShell(configStore, serviceRuntime, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:start":

		startFs := flag.NewFlagSet("app:start", flag.ExitOnError)
		startFs.Usage = func() {
			println("Usage: commander app:start [options] [<app>]*\n")
			println("    Starts one or more apps. If no apps are specified, starts all apps.\n")
			println("Options:\n")
			startFs.PrintDefaults()
		}
		startFs.Parse(flag.Args()[1:])

		apps = startFs.Args()

		if len(apps) == 0 {
			acs, err := configStore.ListApps(env)
			if err != nil {
				log.Fatalf("ERROR: Unable to list apps: %s", err)
			}
			for _, ac := range acs {
				apps = append(apps, ac.Name())
			}
		}
		break
	case "app:status":
		// FIXME: undocumented

		statusFs := flag.NewFlagSet("app:status", flag.ExitOnError)
		statusFs.Usage = func() {
			println("Usage: commander app:status [options] [<app>]*\n")
			println("    Lists status of running apps.\n")
			println("Options:\n")
			statusFs.PrintDefaults()
		}
		statusFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		err := discovery.Status(serviceRuntime, configStore, env, pool, hostIP)
		if err != nil {
			log.Fatalf("ERROR: Unable to list app status: %s", err)
		}
		return

	case "app:stop":
		stopFs := flag.NewFlagSet("app:stop", flag.ExitOnError)
		stopFs.Usage = func() {
			println("Usage: commander app:stop [options] [<app>]*\n")
			println("    Stops one or more apps. If no apps are specified, stops all apps.\n")
			println("Options:\n")
			stopFs.PrintDefaults()
		}
		stopFs.Parse(flag.Args()[1:])

		apps = stopFs.Args()

		for _, app := range apps {
			err := serviceRuntime.StopAllMatching(app)
			if err != nil {
				log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
			}
		}
		if len(apps) > 0 {
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	case "app:unassign":
		appFs := flag.NewFlagSet("app:unassign", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:unassign <app>\n")
			println("    Unassign an app to a pool\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppUnassign(configStore, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "hosts":
		hostFs := flag.NewFlagSet("hosts", flag.ExitOnError)
		hostFs.Usage = func() {
			println("Usage: commander hosts\n")
			println("    List hosts in an env and pool\n")
			println("Options:\n")
			hostFs.PrintDefaults()
		}
		err := hostFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()
		ensurePool()

		err = commander.HostsList(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config":
		configFs := flag.NewFlagSet("config", flag.ExitOnError)
		usage := "Usage: commander config <app>"
		configFs.Usage = func() {
			println(usage)
			println("    List config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() != 1 {
			log.Error("ERROR: Missing app name argument")
			log.Printf("Usage: %s", usage)
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigList(configStore, app, env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:get":
		configFs := flag.NewFlagSet("config:get", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY [KEY]*\n")
			println("    Get config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigGet(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:set":
		configFs := flag.NewFlagSet("config:set", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY=VALUE [KEY=VALUE]*\n")
			println("    Set config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigSet(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:unset":
		configFs := flag.NewFlagSet("config:unset", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY [KEY]*\n")
			println("    Unset config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigUnset(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "runtime":
		runtimeFs := flag.NewFlagSet("runtime", flag.ExitOnError)
		runtimeFs.Usage = func() {
			println("Usage: commander runtime\n")
			println("    List container runtime policies\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}
		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		app := ""
		if runtimeFs.NArg() > 0 {
			app = runtimeFs.Args()[0]
		}

		err = commander.RuntimeList(configStore, app, env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "runtime:set":
		var ps int
		var m string
		var c string
		var vhost string
		var port string
		var maint string
		runtimeFs := flag.NewFlagSet("runtime:set", flag.ExitOnError)
		runtimeFs.IntVar(&ps, "ps", 0, "Number of instances to run across all hosts")
		runtimeFs.StringVar(&m, "m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
		runtimeFs.StringVar(&c, "c", "", "CPU shares (relative weight)")
		runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
		runtimeFs.StringVar(&port, "port", "", "Service port for service discovery")
		runtimeFs.StringVar(&maint, "maint", "", "Enable or disable maintenance mode")

		runtimeFs.Usage = func() {
			println("Usage: commander runtime:set [-ps 1] [-m 100m] [-c 512] [-vhost x.y.z] [-port 8000] [-maint false] <app>\n")
			println("    Set container runtime policies\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}

		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if ps != 0 || m != "" || c != "" || maint != "" {
			ensurePool()
		}

		if runtimeFs.NArg() != 1 {
			runtimeFs.Usage()
			os.Exit(1)
		}

		app := runtimeFs.Args()[0]

		_, err = utils.ParseMemory(m)
		if err != nil {
			log.Fatalf("ERROR: Bad memory option %s: %s", m, err)
		}

		updated, err := commander.RuntimeSet(configStore, app, env, pool, commander.RuntimeOptions{
			Ps:              ps,
			Memory:          m,
			CPUShares:       c,
			VirtualHost:     vhost,
			Port:            port,
			MaintenanceMode: maint,
		})
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}

		if !updated {
			log.Fatalf("ERROR: Failed to set runtime options.")
		}

		if pool != "" {
			log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
		} else {
			log.Printf("Runtime options updated for %s in %s", app, env)
		}
		return

	case "runtime:unset":
		var ps, m, c, port bool
		var vhost string
		runtimeFs := flag.NewFlagSet("runtime:unset", flag.ExitOnError)
		runtimeFs.BoolVar(&ps, "ps", false, "Number of instances to run across all hosts")
		runtimeFs.BoolVar(&m, "m", false, "Memory limit")
		runtimeFs.BoolVar(&c, "c", false, "CPU shares (relative weight)")
		runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
		runtimeFs.BoolVar(&port, "port", false, "Service port for service discovery")

		runtimeFs.Usage = func() {
			println("Usage: commander runtime:unset [-ps] [-m] [-c] [-vhost x.y.z] [-port] <app>\n")
			println("    Reset and removes container runtime policies to defaults\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}

		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if ps || m || c {
			ensurePool()
		}

		if runtimeFs.NArg() != 1 {
			runtimeFs.Usage()
			os.Exit(1)
		}

		app := runtimeFs.Args()[0]

		options := commander.RuntimeOptions{
			VirtualHost: vhost,
		}
		if ps {
			options.Ps = -1
		}

		if m {
			options.Memory = "-"
		}

		if c {
			options.CPUShares = "-"
		}

		if port {
			options.Port = "-"
		}

		updated, err := commander.RuntimeUnset(configStore, app, env, pool, options)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}

		if !updated {
			log.Fatalf("ERROR: Failed to set runtime options.")
		}

		if pool != "" {
			log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
		} else {
			log.Printf("Runtime options updated for %s in %s", app, env)
		}
		return

	case "pool":

		err := commander.ListPools(configStore, env)
		if err != nil {
			log.Fatal(err)
		}
		return

	case "pool:create":
		appFs := flag.NewFlagSet("pool:create", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander -env <env> pool:create <pool>\n")
			println("    Create a pool in <env>\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if pool == "" && appFs.NArg() > 0 {
			pool = appFs.Arg(0)
		} else {
			ensurePool()
		}

		err := commander.PoolCreate(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: Could not create pool: %s", err)
		}
		fmt.Println("created pool:", pool)
		return

	case "pool:delete":
		appFs := flag.NewFlagSet("pool:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander -env <env> pool:delete <pool>\n")
			println("    Delete a pool from <env>\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if pool == "" && flag.NArg() > 1 {
			pool = flag.Arg(1)
		} else {
			ensurePool()
		}

		err := commander.PoolDelete(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: Could not delete pool: %s", err)
			return
		}

		fmt.Println("deleted pool:", pool)
		return

	default:
		fmt.Println("Unknown command")
		flag.Usage()
		os.Exit(1)
	}

	ensureEnv()
	ensurePool()

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("env=%s pool=%s host-ip=%s registry=%s shuttle-addr=%s dns=%s cutoff=%ds",
		env, pool, hostIP, registryURL, shuttleAddr, dns, stopCutoff)

	defer func() {
		configStore.DeleteHost(env, pool, config.HostInfo{
			HostIP: hostIP,
		})
	}()

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {
		wg.Add(1)
		go heartbeatHost()

		go discovery.Register(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr)
		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := configStore.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	// TODO: do we still need a WaitGroup?
	wg.Wait()
}