func TestNodeAvailability(t *testing.T) { store := store.NewMemoryStore(nil) assert.NotNil(t, store) defer store.Close() watch, cancel := state.Watch(store.WatchQueue()) defer cancel() SetupCluster(t, store, watch) node1.Status.State = api.NodeStatus_READY node1.Spec.Availability = api.NodeAvailabilityActive // set node1 to drain updateNodeAvailability(t, store, node1, api.NodeAvailabilityDrain) // task should be set to dead observedTask1 := testutils.WatchShutdownTask(t, watch) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") assert.Equal(t, observedTask1.NodeID, "id1") // set node1 to active updateNodeAvailability(t, store, node1, api.NodeAvailabilityActive) // task should be added back observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") assert.Equal(t, observedTask2.NodeID, "id1") }
func TestAddNode(t *testing.T) { store := store.NewMemoryStore(nil) assert.NotNil(t, store) defer store.Close() watch, cancel := state.Watch(store.WatchQueue()) defer cancel() SetupCluster(t, store, watch) addNode(t, store, node2) observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") assert.Equal(t, observedTask2.NodeID, "id2") }
func SetupCluster(t *testing.T, store *store.MemoryStore, watch chan events.Event) *api.Task { ctx := context.Background() // Start the global orchestrator. global := NewGlobalOrchestrator(store) go func() { assert.NoError(t, global.Run(ctx)) }() addService(t, store, service1) testutils.Expect(t, watch, state.EventCreateService{}) testutils.Expect(t, watch, state.EventCommit{}) addNode(t, store, node1) testutils.Expect(t, watch, state.EventCreateNode{}) testutils.Expect(t, watch, state.EventCommit{}) // return task creation from orchestrator return testutils.WatchTaskCreate(t, watch) }
func TestRemoveTask(t *testing.T) { store := store.NewMemoryStore(nil) assert.NotNil(t, store) defer store.Close() watch, cancel := state.Watch(store.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() observedTask1 := SetupCluster(t, store, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") assert.Equal(t, observedTask1.NodeID, "id1") // delete the task deleteTask(t, store, observedTask1) // the task should be recreated observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") assert.Equal(t, observedTask2.NodeID, "id1") }
func TestUpdaterRollback(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() var ( failImage1 uint32 failImage2 uint32 ) watchCreate, cancelCreate := state.Watch(s.WatchQueue(), state.EventCreateTask{}) defer cancelCreate() watchServiceUpdate, cancelServiceUpdate := state.Watch(s.WatchQueue(), state.EventUpdateService{}) defer cancelServiceUpdate() // Fail new tasks the updater tries to run watchUpdate, cancelUpdate := state.Watch(s.WatchQueue(), state.EventUpdateTask{}) defer cancelUpdate() go func() { failedLast := false for { select { case e := <-watchUpdate: task := e.(state.EventUpdateTask).Task if task.DesiredState == task.Status.State { continue } if task.DesiredState == api.TaskStateRunning && task.Status.State != api.TaskStateFailed && task.Status.State != api.TaskStateRunning { err := s.Update(func(tx store.Tx) error { task = store.GetTask(tx, task.ID) // Never fail two image2 tasks in a row, so there's a mix of // failed and successful tasks for the rollback. if task.Spec.GetContainer().Image == "image1" && atomic.LoadUint32(&failImage1) == 1 { task.Status.State = api.TaskStateFailed failedLast = true } else if task.Spec.GetContainer().Image == "image2" && atomic.LoadUint32(&failImage2) == 1 && !failedLast { task.Status.State = api.TaskStateFailed failedLast = true } else { task.Status.State = task.DesiredState failedLast = false } return store.UpdateTask(tx, task) }) assert.NoError(t, err) } else if task.DesiredState > api.TaskStateRunning { err := s.Update(func(tx store.Tx) error { task = store.GetTask(tx, task.ID) task.Status.State = task.DesiredState return store.UpdateTask(tx, task) }) assert.NoError(t, err) } } } }() // Create a service with four replicas specified before the orchestrator // is started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { s1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{ Image: "image1", }, }, Restart: &api.RestartPolicy{ Condition: api.RestartOnNone, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 4, }, }, Update: &api.UpdateConfig{ FailureAction: api.UpdateConfig_ROLLBACK, Parallelism: 1, Delay: *ptypes.DurationProto(10 * time.Millisecond), Monitor: ptypes.DurationProto(500 * time.Millisecond), MaxFailureRatio: 0.4, }, }, } assert.NoError(t, store.CreateService(tx, s1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask := testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") atomic.StoreUint32(&failImage2, 1) // Start a rolling update err = s.Update(func(tx store.Tx) error { s1 := store.GetService(tx, "id1") require.NotNil(t, s1) s1.PreviousSpec = s1.Spec.Copy() s1.UpdateStatus = nil s1.Spec.Task.GetContainer().Image = "image2" assert.NoError(t, store.UpdateService(tx, s1)) return nil }) assert.NoError(t, err) // Should see three tasks started, then a rollback observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") // Should get to the ROLLBACK_STARTED state for { e := <-watchServiceUpdate if e.(state.EventUpdateService).Service.UpdateStatus == nil { continue } if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_STARTED { break } } observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") // Should end up in ROLLBACK_COMPLETED state for { e := <-watchServiceUpdate if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_COMPLETED { break } } atomic.StoreUint32(&failImage1, 1) // Repeat the rolling update but this time fail the tasks that the // rollback creates. It should end up in ROLLBACK_PAUSED. err = s.Update(func(tx store.Tx) error { s1 := store.GetService(tx, "id1") require.NotNil(t, s1) s1.PreviousSpec = s1.Spec.Copy() s1.UpdateStatus = nil s1.Spec.Task.GetContainer().Image = "image2" assert.NoError(t, store.UpdateService(tx, s1)) return nil }) assert.NoError(t, err) // Should see three tasks started, then a rollback observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image2") // Should get to the ROLLBACK_STARTED state for { e := <-watchServiceUpdate if e.(state.EventUpdateService).Service.UpdateStatus == nil { continue } if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_STARTED { break } } observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") observedTask = testutils.WatchTaskCreate(t, watchCreate) assert.Equal(t, observedTask.Status.State, api.TaskStateNew) assert.Equal(t, observedTask.Spec.GetContainer().Image, "image1") // Should end up in ROLLBACK_PAUSED state for { e := <-watchServiceUpdate if e.(state.EventUpdateService).Service.UpdateStatus.State == api.UpdateStatus_ROLLBACK_PAUSED { break } } }
func TestOrchestratorRestartWindow(t *testing.T) { t.Parallel() ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, Task: api.TaskSpec{ Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: gogotypes.DurationProto(100 * time.Millisecond), MaxAttempts: 1, Window: gogotypes.DurationProto(500 * time.Millisecond), }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") observedTask4 := testutils.WatchTaskUpdate(t, watch) after := time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatal("restart delay should have elapsed") } assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Fail the second task. Confirm that it gets restarted. updatedTask2 := observedTask2.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask5 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask5.Status.State, api.TaskStateNew) assert.Equal(t, observedTask5.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name1") observedTask6 := testutils.WatchTaskUpdate(t, watch) // task gets started after a delay testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask6.Status.State, api.TaskStateNew) assert.Equal(t, observedTask6.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name1") // Fail the first instance again. It should not be restarted. updatedTask1 = observedTask3.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(200 * time.Millisecond): } time.Sleep(time.Second) // Fail the second instance again. It should get restarted because // enough time has elapsed since the last restarts. updatedTask2 = observedTask5.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateFailed} before = time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask7 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask7.Status.State, api.TaskStateNew) assert.Equal(t, observedTask7.DesiredState, api.TaskStateReady) observedTask8 := testutils.WatchTaskUpdate(t, watch) after = time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatal("restart delay should have elapsed") } assert.Equal(t, observedTask8.Status.State, api.TaskStateNew) assert.Equal(t, observedTask8.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask8.ServiceAnnotations.Name, "name1") }
func TestOrchestratorRestartDelay(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: gogotypes.DurationProto(100 * time.Millisecond), }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it gets restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status = api.TaskStatus{State: api.TaskStateFailed} before := time.Now() err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) testutils.Expect(t, watch, state.EventCommit{}) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.DesiredState, api.TaskStateReady) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") observedTask4 := testutils.WatchTaskUpdate(t, watch) after := time.Now() // At least 100 ms should have elapsed. Only check the lower bound, // because the system may be slow and it could have taken longer. if after.Sub(before) < 100*time.Millisecond { t.Fatalf("restart delay should have elapsed. Got: %v", after.Sub(before)) } assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.DesiredState, api.TaskStateRunning) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") }
func TestOrchestratorRestartOnNone(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, Restart: &api.RestartPolicy{ Condition: api.RestartOnNone, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail the first task. Confirm that it does not get restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status.State = api.TaskStateFailed err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(100 * time.Millisecond): } // Mark the second task as completed. Confirm that it does not get restarted. updatedTask2 := observedTask2.Copy() updatedTask2.Status = api.TaskStatus{State: api.TaskStateCompleted} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) assert.NoError(t, err) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) select { case <-watch: t.Fatal("got unexpected event") case <-time.After(100 * time.Millisecond): } }
func TestReplicatedOrchestrator(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { s1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, }, } assert.NoError(t, store.CreateService(tx, s1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Create a second service. err = s.Update(func(tx store.Tx) error { s2 := &api.Service{ ID: "id2", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name2", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 1, }, }, }, } assert.NoError(t, store.CreateService(tx, s2)) return nil }) assert.NoError(t, err) observedTask3 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name2") // Update a service to scale it out to 3 instances err = s.Update(func(tx store.Tx) error { s2 := &api.Service{ ID: "id2", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name2", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 3, }, }, }, } assert.NoError(t, store.UpdateService(tx, s2)) return nil }) assert.NoError(t, err) observedTask4 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name2") observedTask5 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask5.Status.State, api.TaskStateNew) assert.Equal(t, observedTask5.ServiceAnnotations.Name, "name2") // Now scale it back down to 1 instance err = s.Update(func(tx store.Tx) error { s2 := &api.Service{ ID: "id2", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name2", }, Task: api.TaskSpec{ Runtime: &api.TaskSpec_Container{ Container: &api.ContainerSpec{}, }, }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 1, }, }, }, } assert.NoError(t, store.UpdateService(tx, s2)) return nil }) assert.NoError(t, err) observedDeletion1 := testutils.WatchTaskDelete(t, watch) assert.Equal(t, observedDeletion1.Status.State, api.TaskStateNew) assert.Equal(t, observedDeletion1.ServiceAnnotations.Name, "name2") observedDeletion2 := testutils.WatchTaskDelete(t, watch) assert.Equal(t, observedDeletion2.Status.State, api.TaskStateNew) assert.Equal(t, observedDeletion2.ServiceAnnotations.Name, "name2") // There should be one remaining task attached to service id2/name2. var liveTasks []*api.Task s.View(func(readTx store.ReadTx) { var tasks []*api.Task tasks, err = store.FindTasks(readTx, store.ByServiceID("id2")) for _, t := range tasks { if t.DesiredState == api.TaskStateRunning { liveTasks = append(liveTasks, t) } } }) assert.NoError(t, err) assert.Len(t, liveTasks, 1) // Delete the remaining task directly. It should be recreated by the // orchestrator. err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.DeleteTask(tx, liveTasks[0].ID)) return nil }) assert.NoError(t, err) observedTask6 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask6.Status.State, api.TaskStateNew) assert.Equal(t, observedTask6.ServiceAnnotations.Name, "name2") // Delete the service. Its remaining task should go away. err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.DeleteService(tx, "id2")) return nil }) assert.NoError(t, err) deletedTask := testutils.WatchTaskDelete(t, watch) assert.Equal(t, deletedTask.Status.State, api.TaskStateNew) assert.Equal(t, deletedTask.ServiceAnnotations.Name, "name2") }
func TestTaskHistory(t *testing.T) { ctx := context.Background() s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() assert.NoError(t, s.Update(func(tx store.Tx) error { store.CreateCluster(tx, &api.Cluster{ ID: identity.NewID(), Spec: api.ClusterSpec{ Annotations: api.Annotations{ Name: store.DefaultClusterName, }, Orchestration: api.OrchestrationConfig{ TaskHistoryRetentionLimit: 2, }, }, }) return nil })) taskReaper := taskreaper.New(s) defer taskReaper.Stop() orchestrator := NewReplicatedOrchestrator(s) defer orchestrator.Stop() watch, cancel := state.Watch(s.WatchQueue() /*state.EventCreateTask{}, state.EventUpdateTask{}*/) defer cancel() // Create a service with two instances specified before the orchestrator is // started. This should result in two tasks when the orchestrator // starts up. err := s.Update(func(tx store.Tx) error { j1 := &api.Service{ ID: "id1", Spec: api.ServiceSpec{ Annotations: api.Annotations{ Name: "name1", }, Mode: &api.ServiceSpec_Replicated{ Replicated: &api.ReplicatedService{ Replicas: 2, }, }, Task: api.TaskSpec{ Restart: &api.RestartPolicy{ Condition: api.RestartOnAny, Delay: ptypes.DurationProto(0), }, }, }, } assert.NoError(t, store.CreateService(tx, j1)) return nil }) assert.NoError(t, err) // Start the orchestrator. go func() { assert.NoError(t, orchestrator.Run(ctx)) }() go taskReaper.Run() observedTask1 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask1.Status.State, api.TaskStateNew) assert.Equal(t, observedTask1.ServiceAnnotations.Name, "name1") observedTask2 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask2.Status.State, api.TaskStateNew) assert.Equal(t, observedTask2.ServiceAnnotations.Name, "name1") // Fail both tasks. They should both get restarted. updatedTask1 := observedTask1.Copy() updatedTask1.Status.State = api.TaskStateFailed updatedTask1.ServiceAnnotations = api.Annotations{Name: "original"} updatedTask2 := observedTask2.Copy() updatedTask2.Status.State = api.TaskStateFailed updatedTask2.ServiceAnnotations = api.Annotations{Name: "original"} err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask1)) assert.NoError(t, store.UpdateTask(tx, updatedTask2)) return nil }) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventUpdateTask{}) testutils.Expect(t, watch, state.EventCommit{}) testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask3 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask3.Status.State, api.TaskStateNew) assert.Equal(t, observedTask3.ServiceAnnotations.Name, "name1") testutils.Expect(t, watch, state.EventUpdateTask{}) observedTask4 := testutils.WatchTaskCreate(t, watch) assert.Equal(t, observedTask4.Status.State, api.TaskStateNew) assert.Equal(t, observedTask4.ServiceAnnotations.Name, "name1") // Fail these replacement tasks. Since TaskHistory is set to 2, this // should cause the oldest tasks for each instance to get deleted. updatedTask3 := observedTask3.Copy() updatedTask3.Status.State = api.TaskStateFailed updatedTask4 := observedTask4.Copy() updatedTask4.Status.State = api.TaskStateFailed err = s.Update(func(tx store.Tx) error { assert.NoError(t, store.UpdateTask(tx, updatedTask3)) assert.NoError(t, store.UpdateTask(tx, updatedTask4)) return nil }) deletedTask1 := testutils.WatchTaskDelete(t, watch) deletedTask2 := testutils.WatchTaskDelete(t, watch) assert.Equal(t, api.TaskStateFailed, deletedTask1.Status.State) assert.Equal(t, "original", deletedTask1.ServiceAnnotations.Name) assert.Equal(t, api.TaskStateFailed, deletedTask2.Status.State) assert.Equal(t, "original", deletedTask2.ServiceAnnotations.Name) var foundTasks []*api.Task s.View(func(tx store.ReadTx) { foundTasks, err = store.FindTasks(tx, store.All) }) assert.NoError(t, err) assert.Len(t, foundTasks, 4) }