func getRunnableSlotSlice(t *testing.T, s *store.MemoryStore, service *api.Service) []orchestrator.Slot { runnable, _, err := orchestrator.GetRunnableAndDeadSlots(s, service.ID) require.NoError(t, err) var runnableSlice []orchestrator.Slot for _, slot := range runnable { runnableSlice = append(runnableSlice, slot) } return runnableSlice }
func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { runningSlots, deadSlots, err := orchestrator.GetRunnableAndDeadSlots(r.store, service.ID) if err != nil { log.G(ctx).WithError(err).Errorf("reconcile failed finding tasks") return } numSlots := len(runningSlots) slotsSlice := make([]orchestrator.Slot, 0, numSlots) for _, slot := range runningSlots { slotsSlice = append(slotsSlice, slot) } deploy := service.Spec.GetMode().(*api.ServiceSpec_Replicated) specifiedSlots := int(deploy.Replicated.Replicas) switch { case specifiedSlots > numSlots: log.G(ctx).Debugf("Service %s was scaled up from %d to %d instances", service.ID, numSlots, specifiedSlots) // Update all current tasks then add missing tasks r.updater.Update(ctx, r.cluster, service, slotsSlice) _, err = r.store.Batch(func(batch *store.Batch) error { r.addTasks(ctx, batch, service, runningSlots, deadSlots, specifiedSlots-numSlots) r.deleteTasksMap(ctx, batch, deadSlots) return nil }) if err != nil { log.G(ctx).WithError(err).Errorf("reconcile batch failed") } case specifiedSlots < numSlots: // Update up to N tasks then remove the extra log.G(ctx).Debugf("Service %s was scaled down from %d to %d instances", service.ID, numSlots, specifiedSlots) // Preferentially remove tasks on the nodes that have the most // copies of this service, to leave a more balanced result. // First sort tasks such that tasks which are currently running // (in terms of observed state) appear before non-running tasks. // This will cause us to prefer to remove non-running tasks, all // other things being equal in terms of node balance. sort.Sort(slotsByRunningState(slotsSlice)) // Assign each task an index that counts it as the nth copy of // of the service on its node (1, 2, 3, ...), and sort the // tasks by this counter value. slotsByNode := make(map[string]int) slotsWithIndices := make(slotsByIndex, 0, numSlots) for _, slot := range slotsSlice { if len(slot) == 1 && slot[0].NodeID != "" { slotsByNode[slot[0].NodeID]++ slotsWithIndices = append(slotsWithIndices, slotWithIndex{slot: slot, index: slotsByNode[slot[0].NodeID]}) } else { slotsWithIndices = append(slotsWithIndices, slotWithIndex{slot: slot, index: -1}) } } sort.Sort(slotsWithIndices) sortedSlots := make([]orchestrator.Slot, 0, numSlots) for _, slot := range slotsWithIndices { sortedSlots = append(sortedSlots, slot.slot) } r.updater.Update(ctx, r.cluster, service, sortedSlots[:specifiedSlots]) _, err = r.store.Batch(func(batch *store.Batch) error { r.deleteTasksMap(ctx, batch, deadSlots) r.deleteTasks(ctx, batch, sortedSlots[specifiedSlots:]) return nil }) if err != nil { log.G(ctx).WithError(err).Errorf("reconcile batch failed") } case specifiedSlots == numSlots: _, err = r.store.Batch(func(batch *store.Batch) error { r.deleteTasksMap(ctx, batch, deadSlots) return nil }) if err != nil { log.G(ctx).WithError(err).Errorf("reconcile batch failed") } // Simple update, no scaling - update all tasks. r.updater.Update(ctx, r.cluster, service, slotsSlice) } }