Пример #1
0
func pullImage(appCfg config.App) (*docker.Image, error) {

	image, err := serviceRuntime.InspectImage(appCfg.Version())
	if image != nil && image.ID == appCfg.VersionID() || appCfg.VersionID() == "" {
		return image, nil
	}

	log.Printf("Pulling %s version %s\n", appCfg.Name(), appCfg.Version())
	image, err = serviceRuntime.PullImage(appCfg.Version(),
		appCfg.VersionID())
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s",
			appCfg.Version(), err)
		return nil, err
	}

	if image.ID != appCfg.VersionID() && len(appCfg.VersionID()) > 12 {
		log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
			appCfg.Version(),
			image.ID[0:12], appCfg.VersionID()[0:12])
		return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", appCfg.VersionID()[0:12]))
	}

	log.Printf("Pulled %s\n", appCfg.Version())
	return image, nil
}
Пример #2
0
func monitorService(changedConfigs chan *config.ConfigChange) {

	for {

		var changedConfig *config.ConfigChange
		select {

		case changedConfig = <-changedConfigs:

			if changedConfig.Error != nil {
				log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
				continue
			}

			if changedConfig.AppConfig == nil {
				continue
			}

			assigned, err := appAssigned(changedConfig.AppConfig.Name())
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.AppConfig.Name(), err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			ch, ok := workerChans[changedConfig.AppConfig.Name()]
			if !ok {
				name := changedConfig.AppConfig.Name()
				ch := make(chan string)
				workerChans[name] = ch
				wg.Add(1)
				go restartContainers(name, ch)
				ch <- "deploy"

				log.Printf("Started new worker for %s\n", name)
				continue
			}

			if changedConfig.Restart {
				log.Printf("Restarting %s", changedConfig.AppConfig.Name())
				ch <- "restart"
			} else {
				ch <- "deploy"
			}
		}
	}

}
Пример #3
0
func unregisterShuttle(configStore *config.Store, env, hostIP, shuttleAddr string) {

	if client == nil {
		return
	}

	registrations, err := configStore.ListRegistrations(env)
	if err != nil {
		log.Errorf("ERROR: Unable to list registrations: %s", err)
		return
	}

	backends := make(map[string]*shuttle.ServiceConfig)

	for _, r := range registrations {

		// Registration for a container on a different host? Skip it.
		if r.ExternalIP != hostIP {
			continue
		}

		// No service ports exposed on the host, skip it.
		if r.ExternalAddr() == "" || r.Port == "" {
			continue
		}

		service := backends[r.Name]
		if service == nil {
			service = &shuttle.ServiceConfig{
				Name:         r.Name,
				VirtualHosts: r.VirtualHosts,
			}
			if r.Port != "" {
				service.Addr = "0.0.0.0:" + r.Port
			}
			backends[r.Name] = service
		}
		b := shuttle.BackendConfig{
			Name: r.ContainerID[0:12],
			Addr: r.ExternalAddr(),
		}
		service.Backends = append(service.Backends, b)
	}

	for _, service := range backends {

		err := client.RemoveService(service.Name)
		if err != nil {
			log.Errorf("ERROR: Unable to remove shuttle service: %s", err)
		}
	}

}
Пример #4
0
func Register(serviceRuntime *runtime.ServiceRuntime, configStore *config.Store, env, pool, hostIP, shuttleAddr string) {
	if shuttleAddr != "" {
		client = shuttle.NewClient(shuttleAddr)
	}

	RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, false)

	containerEvents := make(chan runtime.ContainerEvent)
	err := serviceRuntime.RegisterEvents(env, pool, hostIP, containerEvents)
	if err != nil {
		log.Printf("ERROR: Unable to register docker event listener: %s", err)
	}

	for {

		select {
		case ce := <-containerEvents:
			switch ce.Status {
			case "start":
				reg, err := configStore.RegisterService(env, pool, hostIP, ce.Container)
				if err != nil {
					log.Errorf("ERROR: Unable to register container: %s", err)
					continue
				}

				log.Printf("Registered %s running as %s for %s%s", strings.TrimPrefix(reg.ContainerName, "/"),
					reg.ContainerID[0:12], reg.Name, locationAt(reg))
				registerShuttle(configStore, env, shuttleAddr)
			case "die", "stop":
				reg, err := configStore.UnRegisterService(env, pool, hostIP, ce.Container)
				if err != nil {
					log.Errorf("ERROR: Unable to unregister container: %s", err)
					continue
				}

				if reg != nil {
					log.Printf("Unregistered %s running as %s for %s%s", strings.TrimPrefix(reg.ContainerName, "/"),
						reg.ContainerID[0:12], reg.Name, locationAt(reg))
				}
				RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, true)
				pruneShuttleBackends(configStore, env, shuttleAddr)
			}

		case <-time.After(10 * time.Second):
			RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, true)
			pruneShuttleBackends(configStore, env, shuttleAddr)
		}
	}
}
Пример #5
0
func RegisterAll(serviceRuntime *runtime.ServiceRuntime, configStore *config.Store, env, pool, hostIP, shuttleAddr string, loggedOnce bool) {
	columns := []string{"CONTAINER ID | IMAGE | EXTERNAL | INTERNAL | CREATED | EXPIRES"}

	registrations, err := serviceRuntime.RegisterAll(env, pool, hostIP)
	if err != nil {
		log.Errorf("ERROR: Unable to register containers: %s", err)
		return
	}

	fn := log.Debugf
	if !loggedOnce {
		fn = log.Printf
	}

	for _, registration := range registrations {
		if !loggedOnce || time.Now().Unix()%60 < 10 {
			fn("Registered %s running as %s for %s%s", strings.TrimPrefix(registration.ContainerName, "/"),
				registration.ContainerID[0:12], registration.Name, locationAt(registration))
		}

		columns = append(columns, strings.Join([]string{
			registration.ContainerID[0:12],
			registration.Image,
			registration.ExternalAddr(),
			registration.InternalAddr(),
			utils.HumanDuration(time.Now().Sub(registration.StartedAt)) + " ago",
			"In " + utils.HumanDuration(registration.Expires.Sub(time.Now().UTC())),
		}, " | "))

	}

	registerShuttle(configStore, env, shuttleAddr)
}
Пример #6
0
func (s *ServiceRuntime) StopOldVersion(appCfg config.App, limit int) error {
	containers, err := s.ManagedContainers()
	if err != nil {
		return err
	}

	stopped := 0

	for _, container := range containers {

		if stopped == limit {
			return nil
		}

		env := s.EnvFor(container)
		// Container name does match one that would be started w/ this service config
		if env["GALAXY_APP"] != appCfg.Name() {
			continue
		}

		image, err := s.InspectImage(container.Image)
		if err != nil {
			log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
			continue
		}

		if image == nil {
			log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
			continue

		}

		version := env["GALAXY_VERSION"]

		if version == "" {
			log.Printf("WARNING: %s missing GALAXY_VERSION", appCfg.ContainerName())
		}

		if version != strconv.FormatInt(appCfg.ID(), 10) && version != "" {
			s.stopContainer(container)
			stopped = stopped + 1
		}
	}

	return nil
}
Пример #7
0
// Backup app config to a file or STDOUT
func appBackup(c *cli.Context) {
	initStore(c)

	env := utils.GalaxyEnv(c)
	if env == "" {
		log.Fatal("ERROR: env is required.  Pass --env or set GALAXY_ENV")
	}

	backup := &backupData{
		Time: time.Now(),
	}

	toBackup := c.Args()

	if len(toBackup) == 0 {
		appList, err := configStore.ListApps(env)
		if err != nil {
			log.Fatalf("ERROR: %s\n", err)
		}

		for _, app := range appList {
			toBackup = append(toBackup, app.Name())
		}
	}

	errCount := 0
	for _, app := range toBackup {
		data, err := getAppBackup(app, env)
		if err != nil {
			// log errors and continue
			log.Errorf("ERROR: %s [%s]", err, app)
			errCount++
			continue
		}
		backup.Apps = append(backup.Apps, data)
	}

	if errCount > 0 {
		fmt.Printf("WARNING: backup completed with %d errors\n", errCount)
		defer os.Exit(errCount)
	}

	j, err := json.MarshalIndent(backup, "", "  ")
	if err != nil {
		log.Fatal(err)
	}

	fileName := c.String("file")
	if fileName != "" {
		if err := ioutil.WriteFile(fileName, j, 0666); err != nil {
			log.Fatal(err)
		}
		return
	}

	os.Stdout.Write(j)
}
Пример #8
0
func pullImage(appCfg config.App) (*docker.Image, error) {
	image, err := serviceRuntime.PullImage(appCfg.Version(), appCfg.VersionID())
	if image == nil || err != nil {
		log.Errorf("ERROR: Could not pull image %s: %s", appCfg.Version(), err)
		return nil, err
	}

	log.Printf("Pulled %s version %s\n", appCfg.Name(), appCfg.Version())
	return image, nil
}
Пример #9
0
func (s *ServiceRuntime) PullImage(version, id string) (*docker.Image, error) {
	image, err := s.InspectImage(version)

	if err != nil && err != docker.ErrNoSuchImage {
		return nil, err
	}

	if image != nil && image.ID == id {
		return image, nil
	}

	registry, repository, tag := utils.SplitDockerImage(version)

	// No, pull it down locally
	pullOpts := docker.PullImageOptions{
		Repository:   repository,
		Tag:          tag,
		OutputStream: log.DefaultLogger}

	dockerAuth := findAuth(registry)

	if registry != "" {
		pullOpts.Repository = registry + "/" + repository
	} else {
		pullOpts.Repository = repository
	}
	pullOpts.Registry = registry
	pullOpts.Tag = tag

	retries := 0
	for {
		retries += 1
		err = s.dockerClient.PullImage(pullOpts, dockerAuth)
		if err != nil {

			// Don't retry 404, they'll never succeed
			if err.Error() == "HTTP code: 404" {
				return image, nil
			}

			if retries > 3 {
				return image, err
			}
			log.Errorf("ERROR: error pulling image %s. Attempt %d: %s", version, retries, err)
			continue
		}
		break
	}

	return s.InspectImage(version)

}
Пример #10
0
func (s *ServiceRuntime) StopAllButCurrentVersion(appCfg config.App) error {
	containers, err := s.ManagedContainers()
	if err != nil {
		return err
	}

	for _, container := range containers {

		env := s.EnvFor(container)
		// Container name does match one that would be started w/ this service config
		if env["GALAXY_APP"] != appCfg.Name() {
			continue
		}

		image, err := s.InspectImage(container.Image)
		if err != nil {
			log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
			continue
		}

		if image == nil {
			log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
			continue

		}

		version := env["GALAXY_VERSION"]

		imageDiffers := image.ID != appCfg.VersionID() && appCfg.VersionID() != ""
		versionDiffers := version != strconv.FormatInt(appCfg.ID(), 10) && version != ""

		if imageDiffers || versionDiffers {
			s.stopContainer(container)
		}
	}
	return nil
}
Пример #11
0
func startService(appCfg config.App, logStatus bool) {

	desired, err := commander.Balanced(configStore, hostIP, appCfg.Name(), env, pool)
	if err != nil {
		log.Errorf("ERROR: Could not determine instance count: %s", err)
		return
	}

	running, err := serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		log.Errorf("ERROR: Could not determine running instance count: %s", err)
		return
	}

	for i := 0; i < desired-running; i++ {
		container, err := serviceRuntime.Start(env, pool, appCfg)
		if err != nil {
			log.Errorf("ERROR: Could not start containers: %s", err)
			return
		}

		log.Printf("Started %s version %s as %s\n", appCfg.Name(), appCfg.Version(), container.ID[0:12])

		err = serviceRuntime.StopOldVersion(appCfg, 1)
		if err != nil {
			log.Errorf("ERROR: Could not stop containers: %s", err)
		}
	}

	running, err = serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
	if err != nil {
		log.Errorf("ERROR: Could not determine running instance count: %s", err)
		return
	}

	for i := 0; i < running-desired; i++ {
		err := serviceRuntime.Stop(appCfg)
		if err != nil {
			log.Errorf("ERROR: Could not stop container: %s", err)
		}
	}

	err = serviceRuntime.StopOldVersion(appCfg, -1)
	if err != nil {
		log.Errorf("ERROR: Could not stop old containers: %s", err)
	}

	// check the image version, and log any inconsistencies
	inspectImage(appCfg)
}
Пример #12
0
// Stop any running galaxy containers that are not assigned to us
// TODO: We call ManagedContainers a lot, repeatedly listing and inspecting all containers.
func (s *ServiceRuntime) StopUnassigned(env, pool string) error {
	containers, err := s.ManagedContainers()
	if err != nil {
		return err
	}

	for _, container := range containers {
		name := s.EnvFor(container)["GALAXY_APP"]

		pools, err := s.configStore.ListAssignedPools(env, name)
		if err != nil {
			log.Errorf("ERROR: Unable to list pool assignments for %s: %s", container.Name, err)
			continue
		}

		if len(pools) == 0 || !utils.StringInSlice(pool, pools) {
			log.Warnf("galaxy container %s not assigned to %s/%s", container.Name, env, pool)
			s.stopContainer(container)
		}
	}
	return nil
}
Пример #13
0
func restartContainers(app string, cmdChan chan string) {
	defer wg.Done()
	logOnce := true

	ticker := time.NewTicker(10 * time.Second)

	for {

		select {

		case cmd := <-cmdChan:

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if !assigned {
				continue
			}

			appCfg, err := configStore.GetApp(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}
				continue
			}

			if appCfg.Version() == "" {
				if !loop {
					return
				}
				continue
			}

			if cmd == "deploy" {
				_, err = pullImage(appCfg)
				if err != nil {
					log.Errorf("ERROR: Error pulling image for %s: %s", app, err)
					if !loop {
						return
					}
					continue
				}
				startService(appCfg, logOnce)
			}

			if cmd == "restart" {
				err := serviceRuntime.Stop(appCfg)
				if err != nil {
					log.Errorf("ERROR: Could not stop %s: %s",
						appCfg.Version(), err)
					if !loop {
						return
					}

					startService(appCfg, logOnce)
					continue
				}
			}

			logOnce = false
		case <-ticker.C:

			appCfg, err := configStore.GetApp(app, env)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				continue
			}

			assigned, err := appAssigned(app)
			if err != nil {
				log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
				if !loop {
					return
				}

				continue
			}

			if appCfg == nil || !assigned {
				log.Errorf("%s no longer exists.  Stopping worker.", app)
				serviceRuntime.StopAllMatching(app)
				delete(workerChans, app)
				return
			}

			if appCfg.Version() == "" {
				continue
			}

			startService(appCfg, logOnce)
		}

		if !loop {
			return
		}

	}
}
Пример #14
0
// RegisterEvents monitors the docker daemon for events, and returns those
// that require registration action over the listener chan.
func (s *ServiceRuntime) RegisterEvents(env, pool, hostIP string, listener chan ContainerEvent) error {
	go func() {
		c := make(chan *docker.APIEvents)

		watching := false
		for {

			err := s.Ping()
			if err != nil {
				log.Errorf("ERROR: Unable to ping docker daemaon: %s", err)
				if watching {
					s.dockerClient.RemoveEventListener(c)
					watching = false
				}
				time.Sleep(10 * time.Second)
				continue

			}

			if !watching {
				err = s.dockerClient.AddEventListener(c)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("ERROR: Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
			}

			select {

			case e := <-c:
				if e.Status == "start" || e.Status == "stop" || e.Status == "die" {
					container, err := s.InspectContainer(e.ID)
					if err != nil {
						log.Printf("ERROR: Error inspecting container: %s", err)
						continue
					}

					if container == nil {
						log.Printf("WARN: Nil container returned for %s", e.ID[:12])
						continue
					}

					name := s.EnvFor(container)["GALAXY_APP"]
					if name != "" {
						registration, err := s.configStore.GetServiceRegistration(env, pool, hostIP, container)
						if err != nil {
							log.Printf("WARN: Could not find service registration for %s/%s: %s", name, container.ID[:12], err)
							continue
						}

						if registration == nil && e.Status != "start" {
							continue
						}

						// if a container is restarting, don't continue re-registering the app
						if container.State.Restarting {
							continue
						}

						listener <- ContainerEvent{
							Status:              e.Status,
							Container:           container,
							ServiceRegistration: registration,
						}
					}

				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}()
	return nil
}
Пример #15
0
func main() {
	flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
	flag.StringVar(&registryURL, "registry", utils.GetEnv("GALAXY_REGISTRY_URL", "redis://127.0.0.1:6379"), "registry URL")
	flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
	flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
	flag.StringVar(&hostIP, "host-ip", "127.0.0.1", "Host IP")
	flag.StringVar(&shuttleAddr, "shuttle-addr", "", "Shuttle API addr (127.0.0.1:9090)")
	flag.StringVar(&dns, "dns", "", "DNS addr to use for containers")
	flag.BoolVar(&debug, "debug", false, "verbose logging")
	flag.BoolVar(&version, "v", false, "display version info")

	flag.Usage = func() {
		println("Usage: commander [options] <command> [<args>]\n")
		println("Available commands are:")
		println("   agent           Runs commander agent")
		println("   app             List all apps")
		println("   app:assign      Assign an app to a pool")
		println("   app:create      Create an app")
		println("   app:deploy      Deploy an app")
		println("   app:delete      Delete an app")
		println("   app:restart     Restart an app")
		println("   app:run         Run a command within an app on this host")
		println("   app:shell       Run a shell within an app on this host")
		println("   app:start       Starts one or more apps")
		println("   app:stop        Stops one or more apps")
		println("   app:unassign    Unassign an app from a pool")
		println("   config          List config for an app")
		println("   config:get      Get config values for an app")
		println("   config:set      Set config values for an app")
		println("   config:unset    Unset config values for an app")
		println("   runtime         List container runtime policies")
		println("   runtime:set     Set container runtime policies")
		println("   hosts           List hosts in an env and pool")
		println("\nOptions:\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if version {
		fmt.Println(buildVersion)
		return
	}

	log.DefaultLogger = log.New(os.Stdout, "", log.INFO)
	log.DefaultLogger.SetFlags(0)

	if debug {
		log.DefaultLogger.Level = log.DEBUG
	}

	if flag.NArg() < 1 {
		fmt.Println("Need a command")
		flag.Usage()
		os.Exit(1)
	}

	initOrDie()

	switch flag.Args()[0] {
	case "dump":
		if flag.NArg() < 2 {
			fmt.Println("Usage: commander dump ENV")
			os.Exit(1)
		}
		dump(flag.Arg(1))
		return

	case "restore":
		if flag.NArg() < 2 {
			fmt.Println("Usage: commander dump ENV FILE")
			os.Exit(1)
		}
		restore(flag.Arg(1))
		return

	case "agent":
		log.DefaultLogger.SetFlags(golog.LstdFlags)
		loop = true
		agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
		agentFs.Usage = func() {
			println("Usage: commander agent [options]\n")
			println("    Runs commander continuously\n\n")
			println("Options:\n\n")
			agentFs.PrintDefaults()
		}
		agentFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

	case "app":
		appFs := flag.NewFlagSet("app", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app\n")
			println("    List all apps or apps in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])
		err := commander.AppList(configStore, env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:assign":
		appFs := flag.NewFlagSet("app:assign", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:assign <app>\n")
			println("    Assign an app to a pool\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppAssign(configStore, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "app:create":
		appFs := flag.NewFlagSet("app:create", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:create <app>\n")
			println("    Create an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppCreate(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:delete":
		appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:delete <app>\n")
			println("    Delete an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppDelete(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:deploy":
		appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:deploy [-force] <app> <version>\n")
			println("    Deploy an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() != 2 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppDeploy(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:restart":
		appFs := flag.NewFlagSet("app:restart", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:restart <app>\n")
			println("    Restart an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() == 0 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppRestart(configStore, appFs.Args()[0], env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:run":
		appFs := flag.NewFlagSet("app:run", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:run <app> <cmd>\n")
			println("    Restart an app in an environment\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if appFs.NArg() < 2 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppRun(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:shell":
		appFs := flag.NewFlagSet("app:shell", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:shell <app>\n")
			println("    Run a shell for an app\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppShell(configStore, serviceRuntime, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "app:start":

		startFs := flag.NewFlagSet("app:start", flag.ExitOnError)
		startFs.Usage = func() {
			println("Usage: commander app:start [options] [<app>]*\n")
			println("    Starts one or more apps. If no apps are specified, starts all apps.\n")
			println("Options:\n")
			startFs.PrintDefaults()
		}
		startFs.Parse(flag.Args()[1:])

		apps = startFs.Args()

		if len(apps) == 0 {
			acs, err := configStore.ListApps(env)
			if err != nil {
				log.Fatalf("ERROR: Unable to list apps: %s", err)
			}
			for _, ac := range acs {
				apps = append(apps, ac.Name())
			}
		}
		break
	case "app:status":
		// FIXME: undocumented

		statusFs := flag.NewFlagSet("app:status", flag.ExitOnError)
		statusFs.Usage = func() {
			println("Usage: commander app:status [options] [<app>]*\n")
			println("    Lists status of running apps.\n")
			println("Options:\n")
			statusFs.PrintDefaults()
		}
		statusFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		err := discovery.Status(serviceRuntime, configStore, env, pool, hostIP)
		if err != nil {
			log.Fatalf("ERROR: Unable to list app status: %s", err)
		}
		return

	case "app:stop":
		stopFs := flag.NewFlagSet("app:stop", flag.ExitOnError)
		stopFs.Usage = func() {
			println("Usage: commander app:stop [options] [<app>]*\n")
			println("    Stops one or more apps. If no apps are specified, stops all apps.\n")
			println("Options:\n")
			stopFs.PrintDefaults()
		}
		stopFs.Parse(flag.Args()[1:])

		apps = stopFs.Args()

		for _, app := range apps {
			err := serviceRuntime.StopAllMatching(app)
			if err != nil {
				log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
			}
		}
		if len(apps) > 0 {
			return
		}

		err := serviceRuntime.StopAll(env)
		if err != nil {
			log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
		}
		return
	case "app:unassign":
		appFs := flag.NewFlagSet("app:unassign", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander app:unassign <app>\n")
			println("    Unassign an app to a pool\n")
			println("Options:\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()
		ensurePool()

		if appFs.NArg() != 1 {
			appFs.Usage()
			os.Exit(1)
		}

		err := commander.AppUnassign(configStore, appFs.Args()[0], env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "hosts":
		hostFs := flag.NewFlagSet("hosts", flag.ExitOnError)
		hostFs.Usage = func() {
			println("Usage: commander hosts\n")
			println("    List hosts in an env and pool\n")
			println("Options:\n")
			hostFs.PrintDefaults()
		}
		err := hostFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()
		ensurePool()

		err = commander.HostsList(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config":
		configFs := flag.NewFlagSet("config", flag.ExitOnError)
		usage := "Usage: commander config <app>"
		configFs.Usage = func() {
			println(usage)
			println("    List config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() != 1 {
			log.Error("ERROR: Missing app name argument")
			log.Printf("Usage: %s", usage)
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigList(configStore, app, env)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:get":
		configFs := flag.NewFlagSet("config:get", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY [KEY]*\n")
			println("    Get config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigGet(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:set":
		configFs := flag.NewFlagSet("config:set", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY=VALUE [KEY=VALUE]*\n")
			println("    Set config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigSet(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return
	case "config:unset":
		configFs := flag.NewFlagSet("config:unset", flag.ExitOnError)
		configFs.Usage = func() {
			println("Usage: commander config <app> KEY [KEY]*\n")
			println("    Unset config values for an app\n")
			println("Options:\n")
			configFs.PrintDefaults()
		}
		err := configFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if configFs.NArg() == 0 {
			log.Errorf("ERROR: Missing app name")
			configFs.Usage()
			os.Exit(1)
		}
		app := configFs.Args()[0]

		err = commander.ConfigUnset(configStore, app, env, configFs.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "runtime":
		runtimeFs := flag.NewFlagSet("runtime", flag.ExitOnError)
		runtimeFs.Usage = func() {
			println("Usage: commander runtime\n")
			println("    List container runtime policies\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}
		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		app := ""
		if runtimeFs.NArg() > 0 {
			app = runtimeFs.Args()[0]
		}

		err = commander.RuntimeList(configStore, app, env, pool)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}
		return

	case "runtime:set":
		var ps int
		var m string
		var c string
		var vhost string
		var port string
		var maint string
		runtimeFs := flag.NewFlagSet("runtime:set", flag.ExitOnError)
		runtimeFs.IntVar(&ps, "ps", 0, "Number of instances to run across all hosts")
		runtimeFs.StringVar(&m, "m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
		runtimeFs.StringVar(&c, "c", "", "CPU shares (relative weight)")
		runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
		runtimeFs.StringVar(&port, "port", "", "Service port for service discovery")
		runtimeFs.StringVar(&maint, "maint", "", "Enable or disable maintenance mode")

		runtimeFs.Usage = func() {
			println("Usage: commander runtime:set [-ps 1] [-m 100m] [-c 512] [-vhost x.y.z] [-port 8000] [-maint false] <app>\n")
			println("    Set container runtime policies\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}

		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if ps != 0 || m != "" || c != "" || maint != "" {
			ensurePool()
		}

		if runtimeFs.NArg() != 1 {
			runtimeFs.Usage()
			os.Exit(1)
		}

		app := runtimeFs.Args()[0]

		_, err = utils.ParseMemory(m)
		if err != nil {
			log.Fatalf("ERROR: Bad memory option %s: %s", m, err)
		}

		updated, err := commander.RuntimeSet(configStore, app, env, pool, commander.RuntimeOptions{
			Ps:              ps,
			Memory:          m,
			CPUShares:       c,
			VirtualHost:     vhost,
			Port:            port,
			MaintenanceMode: maint,
		})
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}

		if !updated {
			log.Fatalf("ERROR: Failed to set runtime options.")
		}

		if pool != "" {
			log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
		} else {
			log.Printf("Runtime options updated for %s in %s", app, env)
		}
		return

	case "runtime:unset":
		var ps, m, c, port bool
		var vhost string
		runtimeFs := flag.NewFlagSet("runtime:unset", flag.ExitOnError)
		runtimeFs.BoolVar(&ps, "ps", false, "Number of instances to run across all hosts")
		runtimeFs.BoolVar(&m, "m", false, "Memory limit")
		runtimeFs.BoolVar(&c, "c", false, "CPU shares (relative weight)")
		runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
		runtimeFs.BoolVar(&port, "port", false, "Service port for service discovery")

		runtimeFs.Usage = func() {
			println("Usage: commander runtime:unset [-ps] [-m] [-c] [-vhost x.y.z] [-port] <app>\n")
			println("    Reset and removes container runtime policies to defaults\n")
			println("Options:\n")
			runtimeFs.PrintDefaults()
		}

		err := runtimeFs.Parse(flag.Args()[1:])
		if err != nil {
			log.Fatalf("ERROR: Bad command line options: %s", err)
		}

		ensureEnv()

		if ps || m || c {
			ensurePool()
		}

		if runtimeFs.NArg() != 1 {
			runtimeFs.Usage()
			os.Exit(1)
		}

		app := runtimeFs.Args()[0]

		options := commander.RuntimeOptions{
			VirtualHost: vhost,
		}
		if ps {
			options.Ps = -1
		}

		if m {
			options.Memory = "-"
		}

		if c {
			options.CPUShares = "-"
		}

		if port {
			options.Port = "-"
		}

		updated, err := commander.RuntimeUnset(configStore, app, env, pool, options)
		if err != nil {
			log.Fatalf("ERROR: %s", err)
		}

		if !updated {
			log.Fatalf("ERROR: Failed to set runtime options.")
		}

		if pool != "" {
			log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
		} else {
			log.Printf("Runtime options updated for %s in %s", app, env)
		}
		return

	case "pool":

		err := commander.ListPools(configStore, env)
		if err != nil {
			log.Fatal(err)
		}
		return

	case "pool:create":
		appFs := flag.NewFlagSet("pool:create", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander -env <env> pool:create <pool>\n")
			println("    Create a pool in <env>\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if pool == "" && appFs.NArg() > 0 {
			pool = appFs.Arg(0)
		} else {
			ensurePool()
		}

		err := commander.PoolCreate(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: Could not create pool: %s", err)
		}
		fmt.Println("created pool:", pool)
		return

	case "pool:delete":
		appFs := flag.NewFlagSet("pool:delete", flag.ExitOnError)
		appFs.Usage = func() {
			println("Usage: commander -env <env> pool:delete <pool>\n")
			println("    Delete a pool from <env>\n")
			appFs.PrintDefaults()
		}
		appFs.Parse(flag.Args()[1:])

		ensureEnv()

		if pool == "" && flag.NArg() > 1 {
			pool = flag.Arg(1)
		} else {
			ensurePool()
		}

		err := commander.PoolDelete(configStore, env, pool)
		if err != nil {
			log.Fatalf("ERROR: Could not delete pool: %s", err)
			return
		}

		fmt.Println("deleted pool:", pool)
		return

	default:
		fmt.Println("Unknown command")
		flag.Usage()
		os.Exit(1)
	}

	ensureEnv()
	ensurePool()

	log.Printf("Starting commander %s", buildVersion)
	log.Printf("env=%s pool=%s host-ip=%s registry=%s shuttle-addr=%s dns=%s cutoff=%ds",
		env, pool, hostIP, registryURL, shuttleAddr, dns, stopCutoff)

	defer func() {
		configStore.DeleteHost(env, pool, config.HostInfo{
			HostIP: hostIP,
		})
	}()

	for app, ch := range workerChans {
		if len(apps) == 0 || utils.StringInSlice(app, apps) {
			wg.Add(1)
			go restartContainers(app, ch)
			ch <- "deploy"
		}
	}

	if loop {
		wg.Add(1)
		go heartbeatHost()

		go discovery.Register(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr)
		cancelChan := make(chan struct{})
		// do we need to cancel ever?

		restartChan := configStore.Watch(env, cancelChan)
		monitorService(restartChan)
	}

	// TODO: do we still need a WaitGroup?
	wg.Wait()
}
Пример #16
0
func pruneShuttleBackends(configStore *config.Store, env, shuttleAddr string) {
	if client == nil {
		return
	}

	config, err := client.GetConfig()
	if err != nil {
		log.Errorf("ERROR: Unable to get shuttle config: %s", err)
		return
	}

	registrations, err := configStore.ListRegistrations(env)
	if err != nil {
		log.Errorf("ERROR: Unable to list registrations: %s", err)
		return
	}

	// FIXME: THERE SHOULD HAVE BEEN AN ERROR IF `len(registrations) == 0` IS WRONG!
	if len(registrations) == 0 {
		// If there are no registrations, skip pruning it because we might be in a bad state and
		// don't want to inadvertently unregister everything.  Shuttle will handle the down
		// nodes if they are really down.
		return
	}

	for _, service := range config.Services {

		app, err := configStore.GetApp(service.Name, env)
		if err != nil {
			log.Errorf("ERROR: Unable to load app %s: %s", app, err)
			continue
		}

		pools, err := configStore.ListAssignedPools(env, service.Name)
		if err != nil {
			log.Errorf("ERROR: Unable to list pool assignments for %s: %s", service.Name, err)
			continue
		}

		if app == nil || len(pools) == 0 {
			err := client.RemoveService(service.Name)
			if err != nil {
				log.Errorf("ERROR: Unable to remove service %s from shuttle: %s", service.Name, err)
			}
			log.Printf("Unregisterred shuttle service %s", service.Name)
			continue
		}

		for _, backend := range service.Backends {
			backendExists := false
			for _, r := range registrations {
				if backend.Name == r.ContainerID[0:12] {
					backendExists = true
					break
				}
			}

			if !backendExists {
				err := client.RemoveBackend(service.Name, backend.Name)
				if err != nil {
					log.Errorf("ERROR: Unable to remove backend %s from shuttle: %s", backend.Name, err)
				}
				log.Printf("Unregisterred shuttle backend %s", backend.Name)
			}
		}
	}
}
Пример #17
0
// restore an app's config from backup
func appRestore(c *cli.Context) {
	initStore(c)

	var err error
	var rawBackup []byte

	fileName := c.String("file")
	if fileName != "" {
		rawBackup, err = ioutil.ReadFile(fileName)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		log.Println("Reading backup from STDIN")
		rawBackup, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
	}

	backup := &backupData{}
	if err := json.Unmarshal(rawBackup, backup); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Found backup from ", backup.Time)

	var toRestore []*appCfg

	if apps := c.Args(); len(apps) > 0 {
		for _, app := range apps {
			found := false
			for _, bkup := range backup.Apps {
				if bkup.Name == app {
					toRestore = append(toRestore, bkup)
					found = true
					break
				}
			}
			if !found {
				log.Fatalf("no backup found for '%s'\n", app)
			}

		}
	} else {
		toRestore = backup.Apps
	}

	// check for conflicts
	// NOTE: there is still a race here if an app is created after this check
	if !c.Bool("force") {
		needForce := false
		for _, bkup := range toRestore {
			exists, err := configStore.AppExists(bkup.Name, utils.GalaxyEnv(c))
			if err != nil {
				log.Fatal(err)
			}
			if exists {
				log.Warnf("Cannot restore over existing app '%s'", bkup.Name)
				needForce = true
			}
		}
		if needForce {
			log.Fatal("Use -force to overwrite")
		}
	}

	loggedErr := false
	for _, bkup := range toRestore {
		if err := restoreApp(bkup, utils.GalaxyEnv(c)); err != nil {
			log.Errorf("%s", err)
			loggedErr = true
		}
	}

	if loggedErr {
		// This is mostly to give a non-zero exit status
		log.Fatal("Error occured during restore")
	}
}
Пример #18
0
func registerShuttle(configStore *config.Store, env, shuttleAddr string) {
	if client == nil {
		return
	}

	registrations, err := configStore.ListRegistrations(env)
	if err != nil {
		log.Errorf("ERROR: Unable to list registrations: %s", err)
		return
	}

	backends := make(map[string]*shuttle.ServiceConfig)

	for _, r := range registrations {

		// No service ports exposed on the host, skip it.
		if r.ExternalAddr() == "" {
			continue
		}

		service := backends[r.Name]
		if service == nil {
			service = &shuttle.ServiceConfig{
				Name:         r.Name,
				VirtualHosts: r.VirtualHosts,
			}
			if r.Port != "" {
				service.Addr = "0.0.0.0:" + r.Port
			}
			backends[r.Name] = service
		}
		b := shuttle.BackendConfig{
			Name:      r.ContainerID[0:12],
			Addr:      r.ExternalAddr(),
			CheckAddr: r.ExternalAddr(),
		}
		service.Backends = append(service.Backends, b)

		// lookup the VIRTUAL_HOST_%d environment variables and load them into the ServiceConfig
		errorPages := make(map[string][]int)
		for vhostCode, url := range r.ErrorPages {
			code := 0
			n, err := fmt.Sscanf(vhostCode, "VIRTUAL_HOST_%d", &code)
			if err != nil || n == 0 {
				continue
			}

			errorPages[url] = append(errorPages[url], code)
		}

		if len(errorPages) > 0 {
			service.ErrorPages = errorPages
		}
	}

	for _, service := range backends {
		err := client.UpdateService(service)
		if err != nil {
			log.Errorf("ERROR: Unable to register shuttle service: %s", err)
		}
	}

}