func (s *Scheduler) handleFormation(ef *ct.ExpandedFormation) (formation *Formation) {
log := logger.New("fn", "handleFormation", "app.id", ef.App.ID, "release.id", ef.Release.ID)
defer func() {
// ensure the formation has the correct omni job counts
if formation.RectifyOmni(len(s.hosts)) {
s.triggerRectify(formation.key())
}
// update any formation-less jobs
if jobs, ok := s.formationlessJobs[formation.key()]; ok {
for _, job := range jobs {
job.Formation = formation
}
s.triggerRectify(formation.key())
delete(s.formationlessJobs, formation.key())
}
}()
formation = s.formations.Get(ef.App.ID, ef.Release.ID)
if formation == nil {
log.Info("adding new formation", "processes", ef.Processes)
formation = s.formations.Add(NewFormation(ef))
} else {
if formation.OriginalProcesses.Equals(ef.Processes) && utils.FormationTagsEqual(formation.Tags, ef.Tags) {
return
} else {
log.Info("updating processes and tags of existing formation", "processes", ef.Processes, "tags", ef.Tags)
formation.Tags = ef.Tags
formation.SetProcesses(ef.Processes)
}
}
s.triggerRectify(formation.key())
return
}
func (s *Scheduler) handleFormation(ef *ct.ExpandedFormation) (formation *Formation) {
log := s.logger.New("fn", "handleFormation", "app.id", ef.App.ID, "release.id", ef.Release.ID)
defer func() {
// ensure the formation has the correct omni job counts
if formation.RectifyOmni(s.activeHostCount()) {
s.triggerRectify(formation.key())
}
// update any formation-less jobs
if jobs, ok := s.formationlessJobs[formation.key()]; ok {
for _, job := range jobs {
job.Formation = formation
}
s.triggerRectify(formation.key())
delete(s.formationlessJobs, formation.key())
}
}()
formation = s.formations.Get(ef.App.ID, ef.Release.ID)
if formation == nil {
log.Info("adding new formation", "processes", ef.Processes)
formation = s.formations.Add(NewFormation(ef))
} else {
diff := Processes(ef.Processes).Diff(formation.OriginalProcesses)
if diff.IsEmpty() && utils.FormationTagsEqual(formation.Tags, ef.Tags) {
return
}
// do not completely scale down critical apps for which this is the only active formation
// (this prevents for example scaling down discoverd which breaks the cluster)
if diff.IsScaleDownOf(formation.OriginalProcesses) && formation.App.Critical() && s.activeFormationCount(formation.App.ID) < 2 {
log.Info("refusing to scale down critical app")
return
}
log.Info("updating processes and tags of existing formation", "processes", ef.Processes, "tags", ef.Tags)
formation.Tags = ef.Tags
formation.SetProcesses(ef.Processes)
}
s.triggerRectify(formation.key())
return
}
// takes args of the form "web=1[,key=val...]", "worker=3[,key=val...]", etc
func runScale(args *docopt.Args, client *controller.Client) error {
app := mustApp()
typeSpecs := args.All["<type>=<spec>"].([]string)
showAll := args.Bool["--all"]
if len(typeSpecs) > 0 && showAll {
return fmt.Errorf("ERROR: Can't use --all when scaling")
}
releaseID := args.String["--release"]
if releaseID != "" && showAll {
return fmt.Errorf("ERROR: Can't use --all in combination with --release")
}
if len(typeSpecs) == 0 {
return showFormations(client, releaseID, showAll, app)
}
release, err := determineRelease(client, releaseID, app)
if err != nil {
return err
}
formation, err := client.GetFormation(app, release.ID)
if err == controller.ErrNotFound {
formation = &ct.Formation{
AppID: app,
ReleaseID: release.ID,
Processes: make(map[string]int),
}
} else if err != nil {
return err
}
if formation.Processes == nil {
formation.Processes = make(map[string]int, len(typeSpecs))
}
if formation.Tags == nil {
formation.Tags = make(map[string]map[string]string, len(typeSpecs))
}
currentProcs := formation.Processes
currentTags := formation.Tags
processes := make(map[string]int, len(currentProcs)+len(typeSpecs))
tags := make(map[string]map[string]string, len(currentTags)+len(typeSpecs))
for k, v := range currentProcs {
processes[k] = v
}
invalid := make([]string, 0, len(release.Processes))
for _, arg := range typeSpecs {
i := strings.IndexRune(arg, '=')
if i < 0 {
return fmt.Errorf("ERROR: scale args must be of the form <typ>=<spec>")
}
countTags := strings.Split(arg[i+1:], ",")
count, err := strconv.Atoi(countTags[0])
if err != nil {
return fmt.Errorf("ERROR: could not parse quantity in %q", arg)
} else if count < 0 {
return fmt.Errorf("ERROR: process quantities cannot be negative in %q", arg)
}
processType := arg[:i]
if _, ok := release.Processes[processType]; ok {
processes[processType] = count
} else {
invalid = append(invalid, fmt.Sprintf("%q", processType))
continue
}
if len(countTags) > 1 {
processTags := make(map[string]string, len(countTags)-1)
for i := 1; i < len(countTags); i++ {
keyVal := strings.SplitN(countTags[i], "=", 2)
if len(keyVal) == 1 && keyVal[0] != "" {
processTags[keyVal[0]] = "true"
} else if len(keyVal) == 2 {
processTags[keyVal[0]] = keyVal[1]
}
}
tags[processType] = processTags
}
}
if len(invalid) > 0 {
return fmt.Errorf("ERROR: unknown process types: %s", strings.Join(invalid, ", "))
}
formation.Processes = processes
formation.Tags = tags
if scalingComplete(currentProcs, processes) {
if !utils.FormationTagsEqual(currentTags, tags) {
// TODO: determine the effect of changing tags and wait
// for appropriate events
fmt.Println("persisting tag change")
return client.PutFormation(formation)
}
fmt.Println("requested scale equals current scale, nothing to do!")
return nil
}
scale := make([]string, 0, len(release.Processes))
for typ := range release.Processes {
if currentProcs[typ] != processes[typ] {
scale = append(scale, fmt.Sprintf("%s: %d=>%d", typ, currentProcs[typ], processes[typ]))
}
}
fmt.Printf("scaling %s\n\n", strings.Join(scale, ", "))
expected := client.ExpectedScalingEvents(currentProcs, processes, release.Processes, 1)
watcher, err := client.WatchJobEvents(app, release.ID)
if err != nil {
return err
}
defer watcher.Close()
err = client.PutFormation(formation)
if err != nil || args.Bool["--no-wait"] {
return err
}
start := time.Now()
err = watcher.WaitFor(expected, scaleTimeout, func(job *ct.Job) error {
id := job.ID
if id == "" {
id = job.UUID
}
fmt.Printf("%s ==> %s %s %s\n", time.Now().Format("15:04:05.000"), job.Type, id, job.State)
return nil
})
if err != nil {
return err
}
fmt.Printf("\nscale completed in %s\n", time.Since(start))
return nil
}
