// removeOldTasks shuts down the given tasks and returns one of the tasks that
// was shut down, or an error.
func (u *Updater) removeOldTasks(ctx context.Context, batch *store.Batch, removeTasks []*api.Task) (*api.Task, error) {
var (
lastErr error
removedTask *api.Task
)
for _, original := range removeTasks {
err := batch.Update(func(tx store.Tx) error {
t := store.GetTask(tx, original.ID)
if t == nil {
return fmt.Errorf("task %s not found while trying to shut it down", original.ID)
}
if t.DesiredState > api.TaskStateRunning {
return fmt.Errorf("task %s was already shut down when reached by updater", original.ID)
}
t.DesiredState = api.TaskStateShutdown
return store.UpdateTask(tx, t)
})
if err != nil {
lastErr = err
} else {
removedTask = original
}
}
if removedTask == nil {
return nil, lastErr
}
return removedTask, nil
}
func (r *ReplicatedOrchestrator) deleteTask(ctx context.Context, batch *store.Batch, t *api.Task) {
err := batch.Update(func(tx store.Tx) error {
return store.DeleteTask(tx, t.ID)
})
if err != nil {
log.G(ctx).WithError(err).Errorf("deleting task %s failed", t.ID)
}
}
func (g *Orchestrator) addTask(ctx context.Context, batch *store.Batch, service *api.Service, nodeID string) {
task := orchestrator.NewTask(g.cluster, service, 0, nodeID)
err := batch.Update(func(tx store.Tx) error {
return store.CreateTask(tx, task)
})
if err != nil {
log.G(ctx).WithError(err).Errorf("global orchestrator: failed to create task")
}
}
func (g *GlobalOrchestrator) removeTask(ctx context.Context, batch *store.Batch, t *api.Task) {
// set existing task DesiredState to TaskStateShutdown
// TODO(aaronl): optimistic update?
err := batch.Update(func(tx store.Tx) error {
t = store.GetTask(tx, t.ID)
if t != nil {
t.DesiredState = api.TaskStateShutdown
return store.UpdateTask(tx, t)
}
return nil
})
if err != nil {
log.G(ctx).WithError(err).Errorf("global orchestrator: removeTask failed to remove %s", t.ID)
}
}
func (a *Allocator) commitAllocatedNetwork(ctx context.Context, batch *store.Batch, n *api.Network) error {
if err := batch.Update(func(tx store.Tx) error {
if err := store.UpdateNetwork(tx, n); err != nil {
return errors.Wrapf(err, "failed updating state in store transaction for network %s", n.ID)
}
return nil
}); err != nil {
if err := a.netCtx.nwkAllocator.Deallocate(n); err != nil {
log.G(ctx).WithError(err).Errorf("failed rolling back allocation of network %s", n.ID)
}
return err
}
return nil
}
func (r *ReplicatedOrchestrator) removeTasks(ctx context.Context, batch *store.Batch, service *api.Service, tasks []*api.Task) {
for _, t := range tasks {
err := batch.Update(func(tx store.Tx) error {
// TODO(aaronl): optimistic update?
t = store.GetTask(tx, t.ID)
if t != nil {
t.DesiredState = api.TaskStateShutdown
return store.UpdateTask(tx, t)
}
return nil
})
if err != nil {
log.G(ctx).WithError(err).Errorf("removing task %s failed", t.ID)
}
}
}
func (a *Allocator) commitAllocatedTask(ctx context.Context, batch *store.Batch, t *api.Task) error {
return batch.Update(func(tx store.Tx) error {
err := store.UpdateTask(tx, t)
if err == store.ErrSequenceConflict {
storeTask := store.GetTask(tx, t.ID)
taskUpdateNetworks(storeTask, t.Networks)
taskUpdateEndpoint(storeTask, t.Endpoint)
if storeTask.Status.State < api.TaskStatePending {
storeTask.Status = t.Status
}
err = store.UpdateTask(tx, storeTask)
}
return errors.Wrapf(err, "failed updating state in store transaction for task %s", t.ID)
})
}
func (u *Updater) removeOldTasks(ctx context.Context, batch *store.Batch, removeTasks []*api.Task) {
for _, original := range removeTasks {
err := batch.Update(func(tx store.Tx) error {
t := store.GetTask(tx, original.ID)
if t == nil {
return fmt.Errorf("task %s not found while trying to shut it down", original.ID)
}
if t.DesiredState > api.TaskStateRunning {
return fmt.Errorf("task %s was already shut down when reached by updater", original.ID)
}
t.DesiredState = api.TaskStateShutdown
return store.UpdateTask(tx, t)
})
if err != nil {
log.G(ctx).WithError(err).Errorf("shutting down stale task %s failed", original.ID)
}
}
}
func (r *ReplicatedOrchestrator) addTasks(ctx context.Context, batch *store.Batch, service *api.Service, runningSlots map[uint64]slot, count int) {
slot := uint64(0)
for i := 0; i < count; i++ {
// Find an slot number that is missing a running task
for {
slot++
if _, ok := runningSlots[slot]; !ok {
break
}
}
err := batch.Update(func(tx store.Tx) error {
return store.CreateTask(tx, newTask(r.cluster, service, slot))
})
if err != nil {
log.G(ctx).Errorf("Failed to create task: %v", err)
}
}
}
func (r *ReplicatedOrchestrator) addTasks(ctx context.Context, batch *store.Batch, service *api.Service, runningInstances map[uint64]struct{}, count int) {
instance := uint64(0)
for i := 0; i < count; i++ {
// Find an instance number that is missing a running task
for {
instance++
if _, ok := runningInstances[instance]; !ok {
break
}
}
err := batch.Update(func(tx store.Tx) error {
return store.CreateTask(tx, newTask(service, instance))
})
if err != nil {
log.G(ctx).Errorf("Failed to create task: %v", err)
}
}
}
func (a *Allocator) commitAllocatedService(ctx context.Context, batch *store.Batch, s *api.Service) error {
if err := batch.Update(func(tx store.Tx) error {
err := store.UpdateService(tx, s)
if err == store.ErrSequenceConflict {
storeService := store.GetService(tx, s.ID)
storeService.Endpoint = s.Endpoint
err = store.UpdateService(tx, storeService)
}
return errors.Wrapf(err, "failed updating state in store transaction for service %s", s.ID)
}); err != nil {
if err := a.netCtx.nwkAllocator.ServiceDeallocate(s); err != nil {
log.G(ctx).WithError(err).Errorf("failed rolling back allocation of service %s", s.ID)
}
return err
}
return nil
}
func (a *Allocator) commitAllocatedNode(ctx context.Context, batch *store.Batch, node *api.Node) error {
if err := batch.Update(func(tx store.Tx) error {
err := store.UpdateNode(tx, node)
if err == store.ErrSequenceConflict {
storeNode := store.GetNode(tx, node.ID)
storeNode.Attachment = node.Attachment.Copy()
err = store.UpdateNode(tx, storeNode)
}
return errors.Wrapf(err, "failed updating state in store transaction for node %s", node.ID)
}); err != nil {
if err := a.netCtx.nwkAllocator.DeallocateNode(node); err != nil {
log.G(ctx).WithError(err).Errorf("failed rolling back allocation of node %s", node.ID)
}
return err
}
return nil
}
