func (u *Updater) worker(ctx context.Context, queue <-chan orchestrator.Slot) {
for slot := range queue {
// Do we have a task with the new spec in desired state = RUNNING?
// If so, all we have to do to complete the update is remove the
// other tasks. Or if we have a task with the new spec that has
// desired state < RUNNING, advance it to running and remove the
// other tasks.
var (
runningTask *api.Task
cleanTask *api.Task
)
for _, t := range slot {
if !u.isTaskDirty(t) {
if t.DesiredState == api.TaskStateRunning {
runningTask = t
break
}
if t.DesiredState < api.TaskStateRunning {
cleanTask = t
}
}
}
if runningTask != nil {
if err := u.useExistingTask(ctx, slot, runningTask); err != nil {
log.G(ctx).WithError(err).Error("update failed")
}
} else if cleanTask != nil {
if err := u.useExistingTask(ctx, slot, cleanTask); err != nil {
log.G(ctx).WithError(err).Error("update failed")
}
} else {
updated := orchestrator.NewTask(u.cluster, u.newService, slot[0].Slot, "")
if orchestrator.IsGlobalService(u.newService) {
updated = orchestrator.NewTask(u.cluster, u.newService, slot[0].Slot, slot[0].NodeID)
}
updated.DesiredState = api.TaskStateReady
if err := u.updateTask(ctx, slot, updated); err != nil {
log.G(ctx).WithError(err).WithField("task.id", updated.ID).Error("update failed")
}
}
if u.newService.Spec.Update != nil && (u.newService.Spec.Update.Delay.Seconds != 0 || u.newService.Spec.Update.Delay.Nanos != 0) {
delay, err := ptypes.Duration(&u.newService.Spec.Update.Delay)
if err != nil {
log.G(ctx).WithError(err).Error("invalid update delay")
continue
}
select {
case <-time.After(delay):
case <-u.stopChan:
return
}
}
}
}
func (g *Orchestrator) addTask(ctx context.Context, batch *store.Batch, service *api.Service, nodeID string) {
task := orchestrator.NewTask(g.cluster, service, 0, nodeID)
err := batch.Update(func(tx store.Tx) error {
return store.CreateTask(tx, task)
})
if err != nil {
log.G(ctx).WithError(err).Errorf("global orchestrator: failed to create task")
}
}
func (r *Orchestrator) addTasks(ctx context.Context, batch *store.Batch, service *api.Service, runningSlots map[uint64]orchestrator.Slot, deadSlots map[uint64]orchestrator.Slot, count int) {
slot := uint64(0)
for i := 0; i < count; i++ {
// Find an slot number that is missing a running task
for {
slot++
if _, ok := runningSlots[slot]; !ok {
break
}
}
delete(deadSlots, slot)
err := batch.Update(func(tx store.Tx) error {
return store.CreateTask(tx, orchestrator.NewTask(r.cluster, service, slot, ""))
})
if err != nil {
log.G(ctx).Errorf("Failed to create task: %v", err)
}
}
}
// Restart initiates a new task to replace t if appropriate under the service's
// restart policy.
func (r *Supervisor) Restart(ctx context.Context, tx store.Tx, cluster *api.Cluster, service *api.Service, t api.Task) error {
// TODO(aluzzardi): This function should not depend on `service`.
// Is the old task still in the process of restarting? If so, wait for
// its restart delay to elapse, to avoid tight restart loops (for
// example, when the image doesn't exist).
r.mu.Lock()
oldDelay, ok := r.delays[t.ID]
if ok {
if !oldDelay.waiter {
oldDelay.waiter = true
go r.waitRestart(ctx, oldDelay, cluster, t.ID)
}
r.mu.Unlock()
return nil
}
r.mu.Unlock()
// Sanity check: was the task shut down already by a separate call to
// Restart? If so, we must avoid restarting it, because this will create
// an extra task. This should never happen unless there is a bug.
if t.DesiredState > api.TaskStateRunning {
return errors.New("Restart called on task that was already shut down")
}
t.DesiredState = api.TaskStateShutdown
err := store.UpdateTask(tx, &t)
if err != nil {
log.G(ctx).WithError(err).Errorf("failed to set task desired state to dead")
return err
}
if !r.shouldRestart(ctx, &t, service) {
return nil
}
var restartTask *api.Task
if orchestrator.IsReplicatedService(service) {
restartTask = orchestrator.NewTask(cluster, service, t.Slot, "")
} else if orchestrator.IsGlobalService(service) {
restartTask = orchestrator.NewTask(cluster, service, 0, t.NodeID)
} else {
log.G(ctx).Error("service not supported by restart supervisor")
return nil
}
n := store.GetNode(tx, t.NodeID)
restartTask.DesiredState = api.TaskStateReady
var restartDelay time.Duration
// Restart delay is not applied to drained nodes
if n == nil || n.Spec.Availability != api.NodeAvailabilityDrain {
if t.Spec.Restart != nil && t.Spec.Restart.Delay != nil {
var err error
restartDelay, err = ptypes.Duration(t.Spec.Restart.Delay)
if err != nil {
log.G(ctx).WithError(err).Error("invalid restart delay; using default")
restartDelay = orchestrator.DefaultRestartDelay
}
} else {
restartDelay = orchestrator.DefaultRestartDelay
}
}
waitStop := true
// Normally we wait for the old task to stop running, but we skip this
// if the old task is already dead or the node it's assigned to is down.
if (n != nil && n.Status.State == api.NodeStatus_DOWN) || t.Status.State > api.TaskStateRunning {
waitStop = false
}
if err := store.CreateTask(tx, restartTask); err != nil {
log.G(ctx).WithError(err).WithField("task.id", restartTask.ID).Error("task create failed")
return err
}
r.recordRestartHistory(restartTask)
r.DelayStart(ctx, tx, &t, restartTask.ID, restartDelay, waitStop)
return nil
}
func TestUpdaterTaskTimeout(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
// Move tasks to their desired state.
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
// Explicitly do not set task state to
// DEAD to trigger TaskTimeout
if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateRunning {
task.Status.State = api.TaskStateRunning
return store.UpdateTask(tx, task)
}
return nil
})
assert.NoError(t, err)
}
}
}()
var instances uint64 = 3
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
},
},
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: instances,
},
},
Update: &api.UpdateConfig{
// avoid having Run block for a long time to watch for failures
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateService(tx, service))
for i := uint64(0); i < instances; i++ {
task := orchestrator.NewTask(nil, service, uint64(i), "")
task.Status.State = api.TaskStateRunning
assert.NoError(t, store.CreateTask(tx, task))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range originalTasks {
for _, task := range slot {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
}
}
before := time.Now()
service.Spec.Task.GetContainer().Image = "v:2"
updater := NewUpdater(s, restart.NewSupervisor(s), nil, service)
// Override the default (1 minute) to speed up the test.
updater.restarts.TaskTimeout = 100 * time.Millisecond
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:2", task.Spec.GetContainer().Image)
}
}
after := time.Now()
// At least 100 ms should have elapsed. Only check the lower bound,
// because the system may be slow and it could have taken longer.
if after.Sub(before) < 100*time.Millisecond {
t.Fatal("stop timeout should have elapsed")
}
}
func TestUpdater(t *testing.T) {
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
// Move tasks to their desired state.
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
if task.Status.State == task.DesiredState {
continue
}
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}()
instances := 3
cluster := &api.Cluster{
// test cluster configuration propagation to task creation.
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: "default",
},
},
}
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: uint64(instances),
},
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
},
},
},
Update: &api.UpdateConfig{
// avoid having Run block for a long time to watch for failures
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateCluster(tx, cluster))
assert.NoError(t, store.CreateService(tx, service))
for i := 0; i < instances; i++ {
assert.NoError(t, store.CreateTask(tx, orchestrator.NewTask(cluster, service, uint64(i), "")))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range originalTasks {
for _, task := range slot {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
assert.Nil(t, task.LogDriver) // should be left alone
}
}
service.Spec.Task.GetContainer().Image = "v:2"
service.Spec.Task.LogDriver = &api.Driver{Name: "tasklogdriver"}
updater := NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:2", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // pick up from task
}
}
service.Spec.Task.GetContainer().Image = "v:3"
cluster.Spec.TaskDefaults.LogDriver = &api.Driver{Name: "clusterlogdriver"} // make cluster default logdriver.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
}
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:3", task.Spec.GetContainer().Image)
assert.Equal(t, service.Spec.Task.LogDriver, task.LogDriver) // still pick up from task
}
}
service.Spec.Task.GetContainer().Image = "v:4"
service.Spec.Task.LogDriver = nil // use cluster default now.
service.Spec.Update = &api.UpdateConfig{
Parallelism: 1,
Delay: 10 * time.Millisecond,
Monitor: gogotypes.DurationProto(50 * time.Millisecond),
}
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
for _, slot := range updatedTasks {
for _, task := range slot {
assert.Equal(t, "v:4", task.Spec.GetContainer().Image)
assert.Equal(t, cluster.Spec.TaskDefaults.LogDriver, task.LogDriver) // pick up from cluster
}
}
}
func TestUpdaterFailureAction(t *testing.T) {
t.Parallel()
ctx := context.Background()
s := store.NewMemoryStore(nil)
assert.NotNil(t, s)
defer s.Close()
// Fail new tasks the updater tries to run
watch, cancel := state.Watch(s.WatchQueue(), state.EventUpdateTask{})
defer cancel()
go func() {
for {
select {
case e := <-watch:
task := e.(state.EventUpdateTask).Task
if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateFailed {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = api.TaskStateFailed
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
} else if task.DesiredState > api.TaskStateRunning {
err := s.Update(func(tx store.Tx) error {
task = store.GetTask(tx, task.ID)
task.Status.State = task.DesiredState
return store.UpdateTask(tx, task)
})
assert.NoError(t, err)
}
}
}
}()
instances := 3
cluster := &api.Cluster{
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: "default",
},
},
}
service := &api.Service{
ID: "id1",
Spec: api.ServiceSpec{
Annotations: api.Annotations{
Name: "name1",
},
Mode: &api.ServiceSpec_Replicated{
Replicated: &api.ReplicatedService{
Replicas: uint64(instances),
},
},
Task: api.TaskSpec{
Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "v:1",
},
},
},
Update: &api.UpdateConfig{
FailureAction: api.UpdateConfig_PAUSE,
Parallelism: 1,
Delay: 500 * time.Millisecond,
Monitor: gogotypes.DurationProto(500 * time.Millisecond),
},
},
}
err := s.Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateCluster(tx, cluster))
assert.NoError(t, store.CreateService(tx, service))
for i := 0; i < instances; i++ {
assert.NoError(t, store.CreateTask(tx, orchestrator.NewTask(cluster, service, uint64(i), "")))
}
return nil
})
assert.NoError(t, err)
originalTasks := getRunnableSlotSlice(t, s, service)
for _, slot := range originalTasks {
for _, task := range slot {
assert.Equal(t, "v:1", task.Spec.GetContainer().Image)
}
}
service.Spec.Task.GetContainer().Image = "v:2"
updater := NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks := getRunnableSlotSlice(t, s, service)
v1Counter := 0
v2Counter := 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:1" {
v1Counter++
} else if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
}
}
}
assert.Equal(t, instances-1, v1Counter)
assert.Equal(t, 1, v2Counter)
s.View(func(tx store.ReadTx) {
service = store.GetService(tx, service.ID)
})
assert.Equal(t, api.UpdateStatus_PAUSED, service.UpdateStatus.State)
// Updating again should do nothing while the update is PAUSED
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
v1Counter = 0
v2Counter = 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:1" {
v1Counter++
} else if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
}
}
}
assert.Equal(t, instances-1, v1Counter)
assert.Equal(t, 1, v2Counter)
// Switch to a service with FailureAction: CONTINUE
err = s.Update(func(tx store.Tx) error {
service = store.GetService(tx, service.ID)
service.Spec.Update.FailureAction = api.UpdateConfig_CONTINUE
service.UpdateStatus = nil
assert.NoError(t, store.UpdateService(tx, service))
return nil
})
assert.NoError(t, err)
service.Spec.Task.GetContainer().Image = "v:3"
updater = NewUpdater(s, restart.NewSupervisor(s), cluster, service)
updater.Run(ctx, getRunnableSlotSlice(t, s, service))
updatedTasks = getRunnableSlotSlice(t, s, service)
v2Counter = 0
v3Counter := 0
for _, slot := range updatedTasks {
for _, task := range slot {
if task.Spec.GetContainer().Image == "v:2" {
v2Counter++
} else if task.Spec.GetContainer().Image == "v:3" {
v3Counter++
}
}
}
assert.Equal(t, 0, v2Counter)
assert.Equal(t, instances, v3Counter)
}