func TestOrchestratorRestartDelay(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: gogotypes.DurationProto(100 * time.Millisecond), }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") observedTask4 := testutils.WatchTaskUpdate(t, watch) after := time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatalf("restart delay should have elapsed. Got: %v", after.Sub(before)) } assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") }
func TestOrchestratorRestartWindow(t *testing.T) { t.Parallel() ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, Task: api.TaskSpec{ Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: gogotypes.DurationProto(100 * time.Millisecond), MaxAttempts: 1, Window: gogotypes.DurationProto(500 * time.Millisecond), }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") observedTask4 := testutils.WatchTaskUpdate(t, watch) after := time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatal("restart delay should have elapsed") } assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Fail the second task. Confirm that it gets restarted. updatedTask2 := observedTask2.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask5 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask5.Status.State, api.TaskStateNew) assert.Equal(t, observedTask5.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name1") observedTask6 := testutils.WatchTaskUpdate(t, watch) // task gets started after a delay testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask6.Status.State, api.TaskStateNew) assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1") // Fail the first instance again. It should not be restarted. updatedTask1 = observedTask3.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(200 * time.Millisecond): } time.Sleep(time.Second) // Fail the second instance again. It should get restarted because // enough time has elapsed since the last restarts. updatedTask2 = observedTask5.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} before = time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask7 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask7.Status.State, api.TaskStateNew) assert.Equal(t, observedTask7.DesiredState, api.TaskStateReady) observedTask8 := testutils.WatchTaskUpdate(t, watch) after = time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatal("restart delay should have elapsed") } assert.Equal(t, observedTask8.Status.State, api.TaskStateNew) assert.Equal(t, observedTask8.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask8.ServiceAnnotations.Name, "name1") }
func TestOrchestratorRestartOnAny(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: gogotypes.DurationProto(0), }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") testutils.Expect(t, watch, state.EventCommit{}) observedTask4 := testutils.WatchTaskUpdate(t, watch) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Mark the second task as completed. Confirm that it gets restarted. updatedTask2 := observedTask2.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask5 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask5.Status.State, api.TaskStateNew) assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name1") testutils.Expect(t, watch, state.EventCommit{}) observedTask6 := testutils.WatchTaskUpdate(t, watch) assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1") }