func pullImage(appCfg config.App) (*docker.Image, error) {
image, err := serviceRuntime.InspectImage(appCfg.Version())
if image != nil && image.ID == appCfg.VersionID() || appCfg.VersionID() == "" {
return image, nil
}
log.Printf("Pulling %s version %s\n", appCfg.Name(), appCfg.Version())
image, err = serviceRuntime.PullImage(appCfg.Version(),
appCfg.VersionID())
if image == nil || err != nil {
log.Errorf("ERROR: Could not pull image %s: %s",
appCfg.Version(), err)
return nil, err
}
if image.ID != appCfg.VersionID() && len(appCfg.VersionID()) > 12 {
log.Errorf("ERROR: Pulled image for %s does not match expected ID. Expected: %s: Got: %s",
appCfg.Version(),
image.ID[0:12], appCfg.VersionID()[0:12])
return nil, errors.New(fmt.Sprintf("failed to pull image ID %s", appCfg.VersionID()[0:12]))
}
log.Printf("Pulled %s\n", appCfg.Version())
return image, nil
}
func monitorService(changedConfigs chan *config.ConfigChange) {
for {
var changedConfig *config.ConfigChange
select {
case changedConfig = <-changedConfigs:
if changedConfig.Error != nil {
log.Errorf("ERROR: Error watching changes: %s", changedConfig.Error)
continue
}
if changedConfig.AppConfig == nil {
continue
}
assigned, err := appAssigned(changedConfig.AppConfig.Name())
if err != nil {
log.Errorf("ERROR: Error retrieving service config for %s: %s", changedConfig.AppConfig.Name(), err)
if !loop {
return
}
continue
}
if !assigned {
continue
}
ch, ok := workerChans[changedConfig.AppConfig.Name()]
if !ok {
name := changedConfig.AppConfig.Name()
ch := make(chan string)
workerChans[name] = ch
wg.Add(1)
go restartContainers(name, ch)
ch <- "deploy"
log.Printf("Started new worker for %s\n", name)
continue
}
if changedConfig.Restart {
log.Printf("Restarting %s", changedConfig.AppConfig.Name())
ch <- "restart"
} else {
ch <- "deploy"
}
}
}
}
func unregisterShuttle(configStore *config.Store, env, hostIP, shuttleAddr string) {
if client == nil {
return
}
registrations, err := configStore.ListRegistrations(env)
if err != nil {
log.Errorf("ERROR: Unable to list registrations: %s", err)
return
}
backends := make(map[string]*shuttle.ServiceConfig)
for _, r := range registrations {
// Registration for a container on a different host? Skip it.
if r.ExternalIP != hostIP {
continue
}
// No service ports exposed on the host, skip it.
if r.ExternalAddr() == "" || r.Port == "" {
continue
}
service := backends[r.Name]
if service == nil {
service = &shuttle.ServiceConfig{
Name: r.Name,
VirtualHosts: r.VirtualHosts,
}
if r.Port != "" {
service.Addr = "0.0.0.0:" + r.Port
}
backends[r.Name] = service
}
b := shuttle.BackendConfig{
Name: r.ContainerID[0:12],
Addr: r.ExternalAddr(),
}
service.Backends = append(service.Backends, b)
}
for _, service := range backends {
err := client.RemoveService(service.Name)
if err != nil {
log.Errorf("ERROR: Unable to remove shuttle service: %s", err)
}
}
}
func Register(serviceRuntime *runtime.ServiceRuntime, configStore *config.Store, env, pool, hostIP, shuttleAddr string) {
if shuttleAddr != "" {
client = shuttle.NewClient(shuttleAddr)
}
RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, false)
containerEvents := make(chan runtime.ContainerEvent)
err := serviceRuntime.RegisterEvents(env, pool, hostIP, containerEvents)
if err != nil {
log.Printf("ERROR: Unable to register docker event listener: %s", err)
}
for {
select {
case ce := <-containerEvents:
switch ce.Status {
case "start":
reg, err := configStore.RegisterService(env, pool, hostIP, ce.Container)
if err != nil {
log.Errorf("ERROR: Unable to register container: %s", err)
continue
}
log.Printf("Registered %s running as %s for %s%s", strings.TrimPrefix(reg.ContainerName, "/"),
reg.ContainerID[0:12], reg.Name, locationAt(reg))
registerShuttle(configStore, env, shuttleAddr)
case "die", "stop":
reg, err := configStore.UnRegisterService(env, pool, hostIP, ce.Container)
if err != nil {
log.Errorf("ERROR: Unable to unregister container: %s", err)
continue
}
if reg != nil {
log.Printf("Unregistered %s running as %s for %s%s", strings.TrimPrefix(reg.ContainerName, "/"),
reg.ContainerID[0:12], reg.Name, locationAt(reg))
}
RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, true)
pruneShuttleBackends(configStore, env, shuttleAddr)
}
case <-time.After(10 * time.Second):
RegisterAll(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr, true)
pruneShuttleBackends(configStore, env, shuttleAddr)
}
}
}
func RegisterAll(serviceRuntime *runtime.ServiceRuntime, configStore *config.Store, env, pool, hostIP, shuttleAddr string, loggedOnce bool) {
columns := []string{"CONTAINER ID | IMAGE | EXTERNAL | INTERNAL | CREATED | EXPIRES"}
registrations, err := serviceRuntime.RegisterAll(env, pool, hostIP)
if err != nil {
log.Errorf("ERROR: Unable to register containers: %s", err)
return
}
fn := log.Debugf
if !loggedOnce {
fn = log.Printf
}
for _, registration := range registrations {
if !loggedOnce || time.Now().Unix()%60 < 10 {
fn("Registered %s running as %s for %s%s", strings.TrimPrefix(registration.ContainerName, "/"),
registration.ContainerID[0:12], registration.Name, locationAt(registration))
}
columns = append(columns, strings.Join([]string{
registration.ContainerID[0:12],
registration.Image,
registration.ExternalAddr(),
registration.InternalAddr(),
utils.HumanDuration(time.Now().Sub(registration.StartedAt)) + " ago",
"In " + utils.HumanDuration(registration.Expires.Sub(time.Now().UTC())),
}, " | "))
}
registerShuttle(configStore, env, shuttleAddr)
}
func (s *ServiceRuntime) StopOldVersion(appCfg config.App, limit int) error {
containers, err := s.ManagedContainers()
if err != nil {
return err
}
stopped := 0
for _, container := range containers {
if stopped == limit {
return nil
}
env := s.EnvFor(container)
// Container name does match one that would be started w/ this service config
if env["GALAXY_APP"] != appCfg.Name() {
continue
}
image, err := s.InspectImage(container.Image)
if err != nil {
log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
continue
}
if image == nil {
log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
continue
}
version := env["GALAXY_VERSION"]
if version == "" {
log.Printf("WARNING: %s missing GALAXY_VERSION", appCfg.ContainerName())
}
if version != strconv.FormatInt(appCfg.ID(), 10) && version != "" {
s.stopContainer(container)
stopped = stopped + 1
}
}
return nil
}
// Backup app config to a file or STDOUT
func appBackup(c *cli.Context) {
initStore(c)
env := utils.GalaxyEnv(c)
if env == "" {
log.Fatal("ERROR: env is required. Pass --env or set GALAXY_ENV")
}
backup := &backupData{
Time: time.Now(),
}
toBackup := c.Args()
if len(toBackup) == 0 {
appList, err := configStore.ListApps(env)
if err != nil {
log.Fatalf("ERROR: %s\n", err)
}
for _, app := range appList {
toBackup = append(toBackup, app.Name())
}
}
errCount := 0
for _, app := range toBackup {
data, err := getAppBackup(app, env)
if err != nil {
// log errors and continue
log.Errorf("ERROR: %s [%s]", err, app)
errCount++
continue
}
backup.Apps = append(backup.Apps, data)
}
if errCount > 0 {
fmt.Printf("WARNING: backup completed with %d errors\n", errCount)
defer os.Exit(errCount)
}
j, err := json.MarshalIndent(backup, "", " ")
if err != nil {
log.Fatal(err)
}
fileName := c.String("file")
if fileName != "" {
if err := ioutil.WriteFile(fileName, j, 0666); err != nil {
log.Fatal(err)
}
return
}
os.Stdout.Write(j)
}
func pullImage(appCfg config.App) (*docker.Image, error) {
image, err := serviceRuntime.PullImage(appCfg.Version(), appCfg.VersionID())
if image == nil || err != nil {
log.Errorf("ERROR: Could not pull image %s: %s", appCfg.Version(), err)
return nil, err
}
log.Printf("Pulled %s version %s\n", appCfg.Name(), appCfg.Version())
return image, nil
}
func (s *ServiceRuntime) PullImage(version, id string) (*docker.Image, error) {
image, err := s.InspectImage(version)
if err != nil && err != docker.ErrNoSuchImage {
return nil, err
}
if image != nil && image.ID == id {
return image, nil
}
registry, repository, tag := utils.SplitDockerImage(version)
// No, pull it down locally
pullOpts := docker.PullImageOptions{
Repository: repository,
Tag: tag,
OutputStream: log.DefaultLogger}
dockerAuth := findAuth(registry)
if registry != "" {
pullOpts.Repository = registry + "/" + repository
} else {
pullOpts.Repository = repository
}
pullOpts.Registry = registry
pullOpts.Tag = tag
retries := 0
for {
retries += 1
err = s.dockerClient.PullImage(pullOpts, dockerAuth)
if err != nil {
// Don't retry 404, they'll never succeed
if err.Error() == "HTTP code: 404" {
return image, nil
}
if retries > 3 {
return image, err
}
log.Errorf("ERROR: error pulling image %s. Attempt %d: %s", version, retries, err)
continue
}
break
}
return s.InspectImage(version)
}
func (s *ServiceRuntime) StopAllButCurrentVersion(appCfg config.App) error {
containers, err := s.ManagedContainers()
if err != nil {
return err
}
for _, container := range containers {
env := s.EnvFor(container)
// Container name does match one that would be started w/ this service config
if env["GALAXY_APP"] != appCfg.Name() {
continue
}
image, err := s.InspectImage(container.Image)
if err != nil {
log.Errorf("ERROR: Unable to inspect image: %s", container.Image)
continue
}
if image == nil {
log.Errorf("ERROR: Image for container %s does not exist!", container.ID[0:12])
continue
}
version := env["GALAXY_VERSION"]
imageDiffers := image.ID != appCfg.VersionID() && appCfg.VersionID() != ""
versionDiffers := version != strconv.FormatInt(appCfg.ID(), 10) && version != ""
if imageDiffers || versionDiffers {
s.stopContainer(container)
}
}
return nil
}
func startService(appCfg config.App, logStatus bool) {
desired, err := commander.Balanced(configStore, hostIP, appCfg.Name(), env, pool)
if err != nil {
log.Errorf("ERROR: Could not determine instance count: %s", err)
return
}
running, err := serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
log.Errorf("ERROR: Could not determine running instance count: %s", err)
return
}
for i := 0; i < desired-running; i++ {
container, err := serviceRuntime.Start(env, pool, appCfg)
if err != nil {
log.Errorf("ERROR: Could not start containers: %s", err)
return
}
log.Printf("Started %s version %s as %s\n", appCfg.Name(), appCfg.Version(), container.ID[0:12])
err = serviceRuntime.StopOldVersion(appCfg, 1)
if err != nil {
log.Errorf("ERROR: Could not stop containers: %s", err)
}
}
running, err = serviceRuntime.InstanceCount(appCfg.Name(), strconv.FormatInt(appCfg.ID(), 10))
if err != nil {
log.Errorf("ERROR: Could not determine running instance count: %s", err)
return
}
for i := 0; i < running-desired; i++ {
err := serviceRuntime.Stop(appCfg)
if err != nil {
log.Errorf("ERROR: Could not stop container: %s", err)
}
}
err = serviceRuntime.StopOldVersion(appCfg, -1)
if err != nil {
log.Errorf("ERROR: Could not stop old containers: %s", err)
}
// check the image version, and log any inconsistencies
inspectImage(appCfg)
}
// Stop any running galaxy containers that are not assigned to us
// TODO: We call ManagedContainers a lot, repeatedly listing and inspecting all containers.
func (s *ServiceRuntime) StopUnassigned(env, pool string) error {
containers, err := s.ManagedContainers()
if err != nil {
return err
}
for _, container := range containers {
name := s.EnvFor(container)["GALAXY_APP"]
pools, err := s.configStore.ListAssignedPools(env, name)
if err != nil {
log.Errorf("ERROR: Unable to list pool assignments for %s: %s", container.Name, err)
continue
}
if len(pools) == 0 || !utils.StringInSlice(pool, pools) {
log.Warnf("galaxy container %s not assigned to %s/%s", container.Name, env, pool)
s.stopContainer(container)
}
}
return nil
}
func restartContainers(app string, cmdChan chan string) {
defer wg.Done()
logOnce := true
ticker := time.NewTicker(10 * time.Second)
for {
select {
case cmd := <-cmdChan:
assigned, err := appAssigned(app)
if err != nil {
log.Errorf("ERROR: Error retrieving assignments for %s: %s", app, err)
if !loop {
return
}
continue
}
if !assigned {
continue
}
appCfg, err := configStore.GetApp(app, env)
if err != nil {
log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
if !loop {
return
}
continue
}
if appCfg.Version() == "" {
if !loop {
return
}
continue
}
if cmd == "deploy" {
_, err = pullImage(appCfg)
if err != nil {
log.Errorf("ERROR: Error pulling image for %s: %s", app, err)
if !loop {
return
}
continue
}
startService(appCfg, logOnce)
}
if cmd == "restart" {
err := serviceRuntime.Stop(appCfg)
if err != nil {
log.Errorf("ERROR: Could not stop %s: %s",
appCfg.Version(), err)
if !loop {
return
}
startService(appCfg, logOnce)
continue
}
}
logOnce = false
case <-ticker.C:
appCfg, err := configStore.GetApp(app, env)
if err != nil {
log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
continue
}
assigned, err := appAssigned(app)
if err != nil {
log.Errorf("ERROR: Error retrieving service config for %s: %s", app, err)
if !loop {
return
}
continue
}
if appCfg == nil || !assigned {
log.Errorf("%s no longer exists. Stopping worker.", app)
serviceRuntime.StopAllMatching(app)
delete(workerChans, app)
return
}
if appCfg.Version() == "" {
continue
}
startService(appCfg, logOnce)
}
if !loop {
return
}
}
}
// RegisterEvents monitors the docker daemon for events, and returns those
// that require registration action over the listener chan.
func (s *ServiceRuntime) RegisterEvents(env, pool, hostIP string, listener chan ContainerEvent) error {
go func() {
c := make(chan *docker.APIEvents)
watching := false
for {
err := s.Ping()
if err != nil {
log.Errorf("ERROR: Unable to ping docker daemaon: %s", err)
if watching {
s.dockerClient.RemoveEventListener(c)
watching = false
}
time.Sleep(10 * time.Second)
continue
}
if !watching {
err = s.dockerClient.AddEventListener(c)
if err != nil && err != docker.ErrListenerAlreadyExists {
log.Printf("ERROR: Error registering docker event listener: %s", err)
time.Sleep(10 * time.Second)
continue
}
watching = true
}
select {
case e := <-c:
if e.Status == "start" || e.Status == "stop" || e.Status == "die" {
container, err := s.InspectContainer(e.ID)
if err != nil {
log.Printf("ERROR: Error inspecting container: %s", err)
continue
}
if container == nil {
log.Printf("WARN: Nil container returned for %s", e.ID[:12])
continue
}
name := s.EnvFor(container)["GALAXY_APP"]
if name != "" {
registration, err := s.configStore.GetServiceRegistration(env, pool, hostIP, container)
if err != nil {
log.Printf("WARN: Could not find service registration for %s/%s: %s", name, container.ID[:12], err)
continue
}
if registration == nil && e.Status != "start" {
continue
}
// if a container is restarting, don't continue re-registering the app
if container.State.Restarting {
continue
}
listener <- ContainerEvent{
Status: e.Status,
Container: container,
ServiceRegistration: registration,
}
}
}
case <-time.After(10 * time.Second):
// check for docker liveness
}
}
}()
return nil
}
func main() {
flag.Int64Var(&stopCutoff, "cutoff", 10, "Seconds to wait before stopping old containers")
flag.StringVar(®istryURL, "registry", utils.GetEnv("GALAXY_REGISTRY_URL", "redis://127.0.0.1:6379"), "registry URL")
flag.StringVar(&env, "env", utils.GetEnv("GALAXY_ENV", ""), "Environment namespace")
flag.StringVar(&pool, "pool", utils.GetEnv("GALAXY_POOL", ""), "Pool namespace")
flag.StringVar(&hostIP, "host-ip", "127.0.0.1", "Host IP")
flag.StringVar(&shuttleAddr, "shuttle-addr", "", "Shuttle API addr (127.0.0.1:9090)")
flag.StringVar(&dns, "dns", "", "DNS addr to use for containers")
flag.BoolVar(&debug, "debug", false, "verbose logging")
flag.BoolVar(&version, "v", false, "display version info")
flag.Usage = func() {
println("Usage: commander [options] <command> [<args>]\n")
println("Available commands are:")
println(" agent Runs commander agent")
println(" app List all apps")
println(" app:assign Assign an app to a pool")
println(" app:create Create an app")
println(" app:deploy Deploy an app")
println(" app:delete Delete an app")
println(" app:restart Restart an app")
println(" app:run Run a command within an app on this host")
println(" app:shell Run a shell within an app on this host")
println(" app:start Starts one or more apps")
println(" app:stop Stops one or more apps")
println(" app:unassign Unassign an app from a pool")
println(" config List config for an app")
println(" config:get Get config values for an app")
println(" config:set Set config values for an app")
println(" config:unset Unset config values for an app")
println(" runtime List container runtime policies")
println(" runtime:set Set container runtime policies")
println(" hosts List hosts in an env and pool")
println("\nOptions:\n")
flag.PrintDefaults()
}
flag.Parse()
if version {
fmt.Println(buildVersion)
return
}
log.DefaultLogger = log.New(os.Stdout, "", log.INFO)
log.DefaultLogger.SetFlags(0)
if debug {
log.DefaultLogger.Level = log.DEBUG
}
if flag.NArg() < 1 {
fmt.Println("Need a command")
flag.Usage()
os.Exit(1)
}
initOrDie()
switch flag.Args()[0] {
case "dump":
if flag.NArg() < 2 {
fmt.Println("Usage: commander dump ENV")
os.Exit(1)
}
dump(flag.Arg(1))
return
case "restore":
if flag.NArg() < 2 {
fmt.Println("Usage: commander dump ENV FILE")
os.Exit(1)
}
restore(flag.Arg(1))
return
case "agent":
log.DefaultLogger.SetFlags(golog.LstdFlags)
loop = true
agentFs := flag.NewFlagSet("agent", flag.ExitOnError)
agentFs.Usage = func() {
println("Usage: commander agent [options]\n")
println(" Runs commander continuously\n\n")
println("Options:\n\n")
agentFs.PrintDefaults()
}
agentFs.Parse(flag.Args()[1:])
ensureEnv()
ensurePool()
case "app":
appFs := flag.NewFlagSet("app", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app\n")
println(" List all apps or apps in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
err := commander.AppList(configStore, env)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:assign":
appFs := flag.NewFlagSet("app:assign", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:assign <app>\n")
println(" Assign an app to a pool\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
ensurePool()
if appFs.NArg() != 1 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppAssign(configStore, appFs.Args()[0], env, pool)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:create":
appFs := flag.NewFlagSet("app:create", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:create <app>\n")
println(" Create an app in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if appFs.NArg() == 0 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppCreate(configStore, appFs.Args()[0], env)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:delete":
appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:delete <app>\n")
println(" Delete an app in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if appFs.NArg() == 0 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppDelete(configStore, appFs.Args()[0], env)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:deploy":
appFs := flag.NewFlagSet("app:delete", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:deploy [-force] <app> <version>\n")
println(" Deploy an app in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if appFs.NArg() != 2 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppDeploy(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1])
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:restart":
appFs := flag.NewFlagSet("app:restart", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:restart <app>\n")
println(" Restart an app in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if appFs.NArg() == 0 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppRestart(configStore, appFs.Args()[0], env)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:run":
appFs := flag.NewFlagSet("app:run", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:run <app> <cmd>\n")
println(" Restart an app in an environment\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if appFs.NArg() < 2 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppRun(configStore, serviceRuntime, appFs.Args()[0], env, appFs.Args()[1:])
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:shell":
appFs := flag.NewFlagSet("app:shell", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:shell <app>\n")
println(" Run a shell for an app\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
ensurePool()
if appFs.NArg() != 1 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppShell(configStore, serviceRuntime, appFs.Args()[0], env, pool)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "app:start":
startFs := flag.NewFlagSet("app:start", flag.ExitOnError)
startFs.Usage = func() {
println("Usage: commander app:start [options] [<app>]*\n")
println(" Starts one or more apps. If no apps are specified, starts all apps.\n")
println("Options:\n")
startFs.PrintDefaults()
}
startFs.Parse(flag.Args()[1:])
apps = startFs.Args()
if len(apps) == 0 {
acs, err := configStore.ListApps(env)
if err != nil {
log.Fatalf("ERROR: Unable to list apps: %s", err)
}
for _, ac := range acs {
apps = append(apps, ac.Name())
}
}
break
case "app:status":
// FIXME: undocumented
statusFs := flag.NewFlagSet("app:status", flag.ExitOnError)
statusFs.Usage = func() {
println("Usage: commander app:status [options] [<app>]*\n")
println(" Lists status of running apps.\n")
println("Options:\n")
statusFs.PrintDefaults()
}
statusFs.Parse(flag.Args()[1:])
ensureEnv()
ensurePool()
err := discovery.Status(serviceRuntime, configStore, env, pool, hostIP)
if err != nil {
log.Fatalf("ERROR: Unable to list app status: %s", err)
}
return
case "app:stop":
stopFs := flag.NewFlagSet("app:stop", flag.ExitOnError)
stopFs.Usage = func() {
println("Usage: commander app:stop [options] [<app>]*\n")
println(" Stops one or more apps. If no apps are specified, stops all apps.\n")
println("Options:\n")
stopFs.PrintDefaults()
}
stopFs.Parse(flag.Args()[1:])
apps = stopFs.Args()
for _, app := range apps {
err := serviceRuntime.StopAllMatching(app)
if err != nil {
log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
}
}
if len(apps) > 0 {
return
}
err := serviceRuntime.StopAll(env)
if err != nil {
log.Fatalf("ERROR: Unable able to stop all containers: %s", err)
}
return
case "app:unassign":
appFs := flag.NewFlagSet("app:unassign", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander app:unassign <app>\n")
println(" Unassign an app to a pool\n")
println("Options:\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
ensurePool()
if appFs.NArg() != 1 {
appFs.Usage()
os.Exit(1)
}
err := commander.AppUnassign(configStore, appFs.Args()[0], env, pool)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "hosts":
hostFs := flag.NewFlagSet("hosts", flag.ExitOnError)
hostFs.Usage = func() {
println("Usage: commander hosts\n")
println(" List hosts in an env and pool\n")
println("Options:\n")
hostFs.PrintDefaults()
}
err := hostFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
ensurePool()
err = commander.HostsList(configStore, env, pool)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "config":
configFs := flag.NewFlagSet("config", flag.ExitOnError)
usage := "Usage: commander config <app>"
configFs.Usage = func() {
println(usage)
println(" List config values for an app\n")
println("Options:\n")
configFs.PrintDefaults()
}
err := configFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if configFs.NArg() != 1 {
log.Error("ERROR: Missing app name argument")
log.Printf("Usage: %s", usage)
os.Exit(1)
}
app := configFs.Args()[0]
err = commander.ConfigList(configStore, app, env)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "config:get":
configFs := flag.NewFlagSet("config:get", flag.ExitOnError)
configFs.Usage = func() {
println("Usage: commander config <app> KEY [KEY]*\n")
println(" Get config values for an app\n")
println("Options:\n")
configFs.PrintDefaults()
}
err := configFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if configFs.NArg() == 0 {
log.Errorf("ERROR: Missing app name")
configFs.Usage()
os.Exit(1)
}
app := configFs.Args()[0]
err = commander.ConfigGet(configStore, app, env, configFs.Args()[1:])
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "config:set":
configFs := flag.NewFlagSet("config:set", flag.ExitOnError)
configFs.Usage = func() {
println("Usage: commander config <app> KEY=VALUE [KEY=VALUE]*\n")
println(" Set config values for an app\n")
println("Options:\n")
configFs.PrintDefaults()
}
err := configFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if configFs.NArg() == 0 {
log.Errorf("ERROR: Missing app name")
configFs.Usage()
os.Exit(1)
}
app := configFs.Args()[0]
err = commander.ConfigSet(configStore, app, env, configFs.Args()[1:])
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "config:unset":
configFs := flag.NewFlagSet("config:unset", flag.ExitOnError)
configFs.Usage = func() {
println("Usage: commander config <app> KEY [KEY]*\n")
println(" Unset config values for an app\n")
println("Options:\n")
configFs.PrintDefaults()
}
err := configFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if configFs.NArg() == 0 {
log.Errorf("ERROR: Missing app name")
configFs.Usage()
os.Exit(1)
}
app := configFs.Args()[0]
err = commander.ConfigUnset(configStore, app, env, configFs.Args()[1:])
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "runtime":
runtimeFs := flag.NewFlagSet("runtime", flag.ExitOnError)
runtimeFs.Usage = func() {
println("Usage: commander runtime\n")
println(" List container runtime policies\n")
println("Options:\n")
runtimeFs.PrintDefaults()
}
err := runtimeFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
app := ""
if runtimeFs.NArg() > 0 {
app = runtimeFs.Args()[0]
}
err = commander.RuntimeList(configStore, app, env, pool)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
return
case "runtime:set":
var ps int
var m string
var c string
var vhost string
var port string
var maint string
runtimeFs := flag.NewFlagSet("runtime:set", flag.ExitOnError)
runtimeFs.IntVar(&ps, "ps", 0, "Number of instances to run across all hosts")
runtimeFs.StringVar(&m, "m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
runtimeFs.StringVar(&c, "c", "", "CPU shares (relative weight)")
runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
runtimeFs.StringVar(&port, "port", "", "Service port for service discovery")
runtimeFs.StringVar(&maint, "maint", "", "Enable or disable maintenance mode")
runtimeFs.Usage = func() {
println("Usage: commander runtime:set [-ps 1] [-m 100m] [-c 512] [-vhost x.y.z] [-port 8000] [-maint false] <app>\n")
println(" Set container runtime policies\n")
println("Options:\n")
runtimeFs.PrintDefaults()
}
err := runtimeFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if ps != 0 || m != "" || c != "" || maint != "" {
ensurePool()
}
if runtimeFs.NArg() != 1 {
runtimeFs.Usage()
os.Exit(1)
}
app := runtimeFs.Args()[0]
_, err = utils.ParseMemory(m)
if err != nil {
log.Fatalf("ERROR: Bad memory option %s: %s", m, err)
}
updated, err := commander.RuntimeSet(configStore, app, env, pool, commander.RuntimeOptions{
Ps: ps,
Memory: m,
CPUShares: c,
VirtualHost: vhost,
Port: port,
MaintenanceMode: maint,
})
if err != nil {
log.Fatalf("ERROR: %s", err)
}
if !updated {
log.Fatalf("ERROR: Failed to set runtime options.")
}
if pool != "" {
log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
} else {
log.Printf("Runtime options updated for %s in %s", app, env)
}
return
case "runtime:unset":
var ps, m, c, port bool
var vhost string
runtimeFs := flag.NewFlagSet("runtime:unset", flag.ExitOnError)
runtimeFs.BoolVar(&ps, "ps", false, "Number of instances to run across all hosts")
runtimeFs.BoolVar(&m, "m", false, "Memory limit")
runtimeFs.BoolVar(&c, "c", false, "CPU shares (relative weight)")
runtimeFs.StringVar(&vhost, "vhost", "", "Virtual host for HTTP routing")
runtimeFs.BoolVar(&port, "port", false, "Service port for service discovery")
runtimeFs.Usage = func() {
println("Usage: commander runtime:unset [-ps] [-m] [-c] [-vhost x.y.z] [-port] <app>\n")
println(" Reset and removes container runtime policies to defaults\n")
println("Options:\n")
runtimeFs.PrintDefaults()
}
err := runtimeFs.Parse(flag.Args()[1:])
if err != nil {
log.Fatalf("ERROR: Bad command line options: %s", err)
}
ensureEnv()
if ps || m || c {
ensurePool()
}
if runtimeFs.NArg() != 1 {
runtimeFs.Usage()
os.Exit(1)
}
app := runtimeFs.Args()[0]
options := commander.RuntimeOptions{
VirtualHost: vhost,
}
if ps {
options.Ps = -1
}
if m {
options.Memory = "-"
}
if c {
options.CPUShares = "-"
}
if port {
options.Port = "-"
}
updated, err := commander.RuntimeUnset(configStore, app, env, pool, options)
if err != nil {
log.Fatalf("ERROR: %s", err)
}
if !updated {
log.Fatalf("ERROR: Failed to set runtime options.")
}
if pool != "" {
log.Printf("Runtime options updated for %s in %s running on %s", app, env, pool)
} else {
log.Printf("Runtime options updated for %s in %s", app, env)
}
return
case "pool":
err := commander.ListPools(configStore, env)
if err != nil {
log.Fatal(err)
}
return
case "pool:create":
appFs := flag.NewFlagSet("pool:create", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander -env <env> pool:create <pool>\n")
println(" Create a pool in <env>\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if pool == "" && appFs.NArg() > 0 {
pool = appFs.Arg(0)
} else {
ensurePool()
}
err := commander.PoolCreate(configStore, env, pool)
if err != nil {
log.Fatalf("ERROR: Could not create pool: %s", err)
}
fmt.Println("created pool:", pool)
return
case "pool:delete":
appFs := flag.NewFlagSet("pool:delete", flag.ExitOnError)
appFs.Usage = func() {
println("Usage: commander -env <env> pool:delete <pool>\n")
println(" Delete a pool from <env>\n")
appFs.PrintDefaults()
}
appFs.Parse(flag.Args()[1:])
ensureEnv()
if pool == "" && flag.NArg() > 1 {
pool = flag.Arg(1)
} else {
ensurePool()
}
err := commander.PoolDelete(configStore, env, pool)
if err != nil {
log.Fatalf("ERROR: Could not delete pool: %s", err)
return
}
fmt.Println("deleted pool:", pool)
return
default:
fmt.Println("Unknown command")
flag.Usage()
os.Exit(1)
}
ensureEnv()
ensurePool()
log.Printf("Starting commander %s", buildVersion)
log.Printf("env=%s pool=%s host-ip=%s registry=%s shuttle-addr=%s dns=%s cutoff=%ds",
env, pool, hostIP, registryURL, shuttleAddr, dns, stopCutoff)
defer func() {
configStore.DeleteHost(env, pool, config.HostInfo{
HostIP: hostIP,
})
}()
for app, ch := range workerChans {
if len(apps) == 0 || utils.StringInSlice(app, apps) {
wg.Add(1)
go restartContainers(app, ch)
ch <- "deploy"
}
}
if loop {
wg.Add(1)
go heartbeatHost()
go discovery.Register(serviceRuntime, configStore, env, pool, hostIP, shuttleAddr)
cancelChan := make(chan struct{})
// do we need to cancel ever?
restartChan := configStore.Watch(env, cancelChan)
monitorService(restartChan)
}
// TODO: do we still need a WaitGroup?
wg.Wait()
}
func pruneShuttleBackends(configStore *config.Store, env, shuttleAddr string) {
if client == nil {
return
}
config, err := client.GetConfig()
if err != nil {
log.Errorf("ERROR: Unable to get shuttle config: %s", err)
return
}
registrations, err := configStore.ListRegistrations(env)
if err != nil {
log.Errorf("ERROR: Unable to list registrations: %s", err)
return
}
// FIXME: THERE SHOULD HAVE BEEN AN ERROR IF `len(registrations) == 0` IS WRONG!
if len(registrations) == 0 {
// If there are no registrations, skip pruning it because we might be in a bad state and
// don't want to inadvertently unregister everything. Shuttle will handle the down
// nodes if they are really down.
return
}
for _, service := range config.Services {
app, err := configStore.GetApp(service.Name, env)
if err != nil {
log.Errorf("ERROR: Unable to load app %s: %s", app, err)
continue
}
pools, err := configStore.ListAssignedPools(env, service.Name)
if err != nil {
log.Errorf("ERROR: Unable to list pool assignments for %s: %s", service.Name, err)
continue
}
if app == nil || len(pools) == 0 {
err := client.RemoveService(service.Name)
if err != nil {
log.Errorf("ERROR: Unable to remove service %s from shuttle: %s", service.Name, err)
}
log.Printf("Unregisterred shuttle service %s", service.Name)
continue
}
for _, backend := range service.Backends {
backendExists := false
for _, r := range registrations {
if backend.Name == r.ContainerID[0:12] {
backendExists = true
break
}
}
if !backendExists {
err := client.RemoveBackend(service.Name, backend.Name)
if err != nil {
log.Errorf("ERROR: Unable to remove backend %s from shuttle: %s", backend.Name, err)
}
log.Printf("Unregisterred shuttle backend %s", backend.Name)
}
}
}
}
// restore an app's config from backup
func appRestore(c *cli.Context) {
initStore(c)
var err error
var rawBackup []byte
fileName := c.String("file")
if fileName != "" {
rawBackup, err = ioutil.ReadFile(fileName)
if err != nil {
log.Fatal(err)
}
} else {
log.Println("Reading backup from STDIN")
rawBackup, err = ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatal(err)
}
}
backup := &backupData{}
if err := json.Unmarshal(rawBackup, backup); err != nil {
log.Fatal(err)
}
fmt.Println("Found backup from ", backup.Time)
var toRestore []*appCfg
if apps := c.Args(); len(apps) > 0 {
for _, app := range apps {
found := false
for _, bkup := range backup.Apps {
if bkup.Name == app {
toRestore = append(toRestore, bkup)
found = true
break
}
}
if !found {
log.Fatalf("no backup found for '%s'\n", app)
}
}
} else {
toRestore = backup.Apps
}
// check for conflicts
// NOTE: there is still a race here if an app is created after this check
if !c.Bool("force") {
needForce := false
for _, bkup := range toRestore {
exists, err := configStore.AppExists(bkup.Name, utils.GalaxyEnv(c))
if err != nil {
log.Fatal(err)
}
if exists {
log.Warnf("Cannot restore over existing app '%s'", bkup.Name)
needForce = true
}
}
if needForce {
log.Fatal("Use -force to overwrite")
}
}
loggedErr := false
for _, bkup := range toRestore {
if err := restoreApp(bkup, utils.GalaxyEnv(c)); err != nil {
log.Errorf("%s", err)
loggedErr = true
}
}
if loggedErr {
// This is mostly to give a non-zero exit status
log.Fatal("Error occured during restore")
}
}
func registerShuttle(configStore *config.Store, env, shuttleAddr string) {
if client == nil {
return
}
registrations, err := configStore.ListRegistrations(env)
if err != nil {
log.Errorf("ERROR: Unable to list registrations: %s", err)
return
}
backends := make(map[string]*shuttle.ServiceConfig)
for _, r := range registrations {
// No service ports exposed on the host, skip it.
if r.ExternalAddr() == "" {
continue
}
service := backends[r.Name]
if service == nil {
service = &shuttle.ServiceConfig{
Name: r.Name,
VirtualHosts: r.VirtualHosts,
}
if r.Port != "" {
service.Addr = "0.0.0.0:" + r.Port
}
backends[r.Name] = service
}
b := shuttle.BackendConfig{
Name: r.ContainerID[0:12],
Addr: r.ExternalAddr(),
CheckAddr: r.ExternalAddr(),
}
service.Backends = append(service.Backends, b)
// lookup the VIRTUAL_HOST_%d environment variables and load them into the ServiceConfig
errorPages := make(map[string][]int)
for vhostCode, url := range r.ErrorPages {
code := 0
n, err := fmt.Sscanf(vhostCode, "VIRTUAL_HOST_%d", &code)
if err != nil || n == 0 {
continue
}
errorPages[url] = append(errorPages[url], code)
}
if len(errorPages) > 0 {
service.ErrorPages = errorPages
}
}
for _, service := range backends {
err := client.UpdateService(service)
if err != nil {
log.Errorf("ERROR: Unable to register shuttle service: %s", err)
}
}
}