func TestStopWithPendingStops(t *testing.T) { ctrl, client, testTime, taskEngine, _, imageManager := mocks(t, &defaultConfig) defer ctrl.Finish() testTime.EXPECT().Now().AnyTimes() testTime.EXPECT().After(gomock.Any()).AnyTimes() sleepTask1 := testdata.LoadTask("sleep5") sleepTask1.StartSequenceNumber = 5 sleepTask2 := testdata.LoadTask("sleep5") sleepTask2.Arn = "arn2" eventStream := make(chan DockerContainerChangeEvent) client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) err := taskEngine.Init() if err != nil { t.Fatal(err) } taskEvents, contEvents := taskEngine.TaskEvents() go func() { for { <-taskEvents } }() go func() { for { <-contEvents } }() pullDone := make(chan bool) pullInvoked := make(chan bool) client.EXPECT().PullImage(gomock.Any(), nil).Do(func(x, y interface{}) { pullInvoked <- true <-pullDone }) imageManager.EXPECT().AddContainerReferenceToImageState(gomock.Any()).AnyTimes() imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).AnyTimes() taskEngine.AddTask(sleepTask2) <-pullInvoked stopSleep2 := *sleepTask2 stopSleep2.SetDesiredStatus(api.TaskStopped) stopSleep2.StopSequenceNumber = 4 taskEngine.AddTask(&stopSleep2) taskEngine.AddTask(sleepTask1) stopSleep1 := *sleepTask1 stopSleep1.SetDesiredStatus(api.TaskStopped) stopSleep1.StopSequenceNumber = 5 taskEngine.AddTask(&stopSleep1) pullDone <- true // this means the PullImage is only called once due to the task is stopped before it // gets the pull image lock }
func TestStopWithPendingStops(t *testing.T) { ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() testTime.EXPECT().Now().AnyTimes() testTime.EXPECT().After(gomock.Any()).AnyTimes() sleepTask1 := testdata.LoadTask("sleep5") sleepTask1.StartSequenceNumber = 5 sleepTask2 := testdata.LoadTask("sleep5") sleepTask2.Arn = "arn2" eventStream := make(chan DockerContainerChangeEvent) client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) err := taskEngine.Init() if err != nil { t.Fatal(err) } taskEvents, contEvents := taskEngine.TaskEvents() go func() { for { <-taskEvents } }() go func() { for { <-contEvents } }() pulling := make(chan bool) client.EXPECT().PullImage(gomock.Any(), nil).Do(func(x, y interface{}) { <-pulling }) taskEngine.AddTask(sleepTask2) stopSleep2 := *sleepTask2 stopSleep2.DesiredStatus = api.TaskStopped stopSleep2.StopSequenceNumber = 4 taskEngine.AddTask(&stopSleep2) taskEngine.AddTask(sleepTask1) stopSleep1 := *sleepTask1 stopSleep1.DesiredStatus = api.TaskStopped stopSleep1.StopSequenceNumber = 5 taskEngine.AddTask(&stopSleep1) pulling <- true // If we get here without deadlocking, we passed the test }
func TestGetTaskByArn(t *testing.T) { // Need a mock client as AddTask not only adds a task to the engine, but // also causes the engine to progress the task. ctrl, client, _, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() eventStream := make(chan DockerContainerChangeEvent) client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) client.EXPECT().PullImage(gomock.Any(), gomock.Any()).AnyTimes() // TODO change to MaxTimes(1) err := taskEngine.Init() if err != nil { t.Fatal(err) } defer taskEngine.Disable() sleepTask := testdata.LoadTask("sleep5") sleepTaskArn := sleepTask.Arn taskEngine.AddTask(sleepTask) _, found := taskEngine.GetTaskByArn(sleepTaskArn) if !found { t.Fatalf("Task %s not found", sleepTaskArn) } _, found = taskEngine.GetTaskByArn(sleepTaskArn + "arn") if found { t.Fatal("Task with invalid arn found in the task engine") } }
func TestCreateContainerForceSave(t *testing.T) { ctrl, client, _, privateTaskEngine, _ := mocks(t, &config.Config{}) saver := mock_statemanager.NewMockStateManager(ctrl) defer ctrl.Finish() taskEngine, _ := privateTaskEngine.(*DockerTaskEngine) taskEngine.SetSaver(saver) sleepTask := testdata.LoadTask("sleep5") sleepContainer, _ := sleepTask.ContainerByName("sleep5") gomock.InOrder( saver.EXPECT().ForceSave().Do(func() interface{} { task, ok := taskEngine.state.TaskByArn(sleepTask.Arn) if task == nil || !ok { t.Fatalf("Expected task with arn %s", sleepTask.Arn) } _, ok = task.ContainerByName("sleep5") if !ok { t.Error("Expected container sleep5") } return nil }), client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()), ) metadata := taskEngine.createContainer(sleepTask, sleepContainer) if metadata.Error != nil { t.Error("Unexpected error", metadata.Error) } }
// TestTaskTransitionWhenStopContainerTimesout tests that task transitions to stopped // only when terminal events are recieved from docker event stream when // StopContainer times out func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { ctrl, client, mockTime, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) mockTime.EXPECT().After(gomock.Any()).AnyTimes() containerStopTimeoutError := DockerContainerMetadata{Error: &DockerTimeoutError{transition: "stop", duration: stopContainerTimeout}} for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } // Container config should get updated with this during CreateContainer dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version dockerConfig.Labels["com.amazonaws.ecs.cluster"] = "" client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) // StartContainer returns timeout error. This should cause the engine // to transition the task to STOPPED and to stop all containers of // the task client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}}) gomock.InOrder( // StopContainer times out as well client.EXPECT().StopContainer("containerId").Do(func(id string) { // Simulate docker actually stopping the container even though // StopContainer in container engine times out go func() { eventStream <- dockerEvent(api.ContainerStopped) }() }).Return(containerStopTimeoutError), // Since task is not in steady state, progressContainers causes // another invocation of StopContainer. Return a timeout error // for that as well // TODO change AnyTimes() to MinTimes(1) after updating gomock client.EXPECT().StopContainer("containerId").Return(containerStopTimeoutError).AnyTimes(), ) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatalf("Error getting event streams from engine: %v", err) } taskEngine.AddTask(sleepTask) // Expect it to go to stopped contEvent := <-contEvents if contEvent.Status != api.ContainerStopped { t.Errorf("Expected container to timeout on start and stop, got: %v", contEvent) } *contEvent.SentStatus = api.ContainerStopped taskEvent := <-taskEvents if taskEvent.Status != api.TaskStopped { t.Errorf("Expected task to be stopped, got: %v", taskEvent) } *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: t.Error("Should be out of events") case <-contEvents: t.Error("Should be out of events") default: } }
func TestBatchContainerHappyPath(t *testing.T) { ctrl, client, mockTime, taskEngine, credentialsManager := mocks(t, &defaultConfig) defer ctrl.Finish() roleCredentials := &credentials.TaskIAMRoleCredentials{ IAMRoleCredentials: credentials.IAMRoleCredentials{CredentialsId: "credsid"}, } credentialsManager.EXPECT().GetTaskCredentials(credentialsId).Return(roleCredentials, true).AnyTimes() credentialsManager.EXPECT().RemoveCredentials(credentialsId) sleepTask := testdata.LoadTask("sleep5") sleepTask.SetCredentialsId(credentialsId) eventStream := make(chan DockerContainerChangeEvent) eventsReported := sync.WaitGroup{} dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) var createdContainerName string for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) // Container config should get updated with this during PostUnmarshalTask credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI() dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue) if err != nil { t.Fatal(err) } // Container config should get updated with this during CreateContainer dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version dockerConfig.Labels["com.amazonaws.ecs.cluster"] = "" client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(config *docker.Config, y interface{}, containerName string) { if !reflect.DeepEqual(dockerConfig, config) { t.Errorf("Mismatch in container config; expected: %v, got: %v", dockerConfig, config) } // sleep5 task contains only one container. Just assign // the containerName to createdContainerName createdContainerName = containerName eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerCreated) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerRunning) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) } steadyStateVerify := make(chan time.Time, 1) cleanup := make(chan time.Time, 1) mockTime.EXPECT().Now().Do(func() time.Time { return time.Now() }).AnyTimes() mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes() mockTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes() err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } eventsReported.Wait() // Wait for all events to be consumed prior to moving it towards stopped; we // don't want to race the below with these or we'll end up with the "going // backwards in state" stop and we haven't 'expect'd for that exitCode := 0 // And then docker reports that sleep died, as sleep is wont to do eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}} steadyStateVerify <- time.Now() if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") if *cont.ExitCode != 0 { t.Fatal("Exit code should be present") } } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } // Extra events should not block forever; duplicate acs and docker events are possible go func() { eventStream <- dockerEvent(api.ContainerStopped) }() go func() { eventStream <- dockerEvent(api.ContainerStopped) }() sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.SetCredentialsId(credentialsId) sleepTaskStop.DesiredStatus = api.TaskStopped taskEngine.AddTask(sleepTaskStop) // As above, duplicate events should not be a problem taskEngine.AddTask(sleepTaskStop) taskEngine.AddTask(sleepTaskStop) // Expect a bunch of steady state 'poll' describes when we trigger cleanup client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer(gomock.Any()).Do(func(removedContainerName string) { if createdContainerName != removedContainerName { t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName) } }).Return(nil) // trigger cleanup cleanup <- time.Now() go func() { eventStream <- dockerEvent(api.ContainerStopped) }() // Wait for the task to actually be dead; if we just fallthrough immediately, // the remove might not have happened (expectation failure) for { tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks() if len(tasks) == 0 { break } time.Sleep(5 * time.Millisecond) } }
func TestSteadyStatePoll(t *testing.T) { ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } // Container config should get updated with this during CreateContainer dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version dockerConfig.Labels["com.amazonaws.ecs.cluster"] = "" client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { go func() { eventStream <- dockerEvent(api.ContainerRunning) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) } steadyStateVerify := make(chan time.Time, 10) testTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes() testTime.EXPECT().After(gomock.Any()).AnyTimes() err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } // Two steady state oks, one stop gomock.InOrder( client.EXPECT().DescribeContainer("containerId").Return(api.ContainerRunning, DockerContainerMetadata{DockerId: "containerId"}).Times(2), // TODO change AnyTimes() to MinTimes(1) after updating gomock client.EXPECT().DescribeContainer("containerId").Return(api.ContainerStopped, DockerContainerMetadata{DockerId: "containerId"}).AnyTimes(), ) for i := 0; i < 10; i++ { steadyStateVerify <- time.Now() } close(steadyStateVerify) contEvent := <-contEvents if contEvent.Status != api.ContainerStopped { t.Error("Expected container to be stopped") } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } // cleanup expectations testTime.EXPECT().Now().AnyTimes() testTime.EXPECT().After(gomock.Any()).AnyTimes() }
func TestStartTimeoutThenStart(t *testing.T) { ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) testTime.EXPECT().After(gomock.Any()) dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } // Container config should get updated with this during CreateContainer dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version dockerConfig.Labels["com.amazonaws.ecs.cluster"] = "" client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}}) // Expect it to try to stop the container before going on; // in the future the agent might optimize to not stop unless the known // status is running, at which point this can be safely removed client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: CannotXContainerError{transition: "start", msg: "Cannot start"}}) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) // Expect it to go to stopped contEvent := <-contEvents if contEvent.Status != api.ContainerStopped { t.Fatal("Expected container to timeout on start and stop") } *contEvent.SentStatus = api.ContainerStopped taskEvent := <-taskEvents if taskEvent.Status != api.TaskStopped { t.Fatal("And then task") } *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } // Expect it to try to stop it once now client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: CannotXContainerError{transition: "start", msg: "Cannot start"}}).AnyTimes() // Now surprise surprise, it actually did start! eventStream <- dockerEvent(api.ContainerRunning) // However, if it starts again, we should not see it be killed; no additional expect eventStream <- dockerEvent(api.ContainerRunning) eventStream <- dockerEvent(api.ContainerRunning) select { case <-taskEvents: t.Fatal("Should be out of events") case ev := <-contEvents: t.Fatal("Should be out of events", ev) default: } }
func TestRemoveEvents(t *testing.T) { ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig) defer ctrl.Finish() sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) eventsReported := sync.WaitGroup{} dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) var createdContainerName string for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(x, y interface{}, containerName string) { // sleep5 task contains only one container. Just assign // the containerName to createdContainerName createdContainerName = containerName eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerCreated) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerRunning) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) } steadyStateVerify := make(chan time.Time, 1) cleanup := make(chan time.Time, 1) testTime.EXPECT().Now().Do(func() time.Time { return time.Now() }).AnyTimes() testTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes() testTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes() err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } eventsReported.Wait() // Wait for all events to be consumed prior to moving it towards stopped; we // don't want to race the below with these or we'll end up with the "going // backwards in state" stop and we haven't 'expect'd for that exitCode := 0 // And then docker reports that sleep died, as sleep is wont to do eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}} steadyStateVerify <- time.Now() if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") if *cont.ExitCode != 0 { t.Fatal("Exit code should be present") } } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.DesiredStatus = api.TaskStopped taskEngine.AddTask(sleepTaskStop) // Expect a bunch of steady state 'poll' describes when we warp 4 hours client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer(gomock.Any()).Do(func(removedContainerName string) { if createdContainerName != removedContainerName { t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName) } // Emit a couple events for the task before the remove finishes; make sure this gets handled appropriately eventStream <- dockerEvent(api.ContainerStopped) eventStream <- dockerEvent(api.ContainerStopped) }).Return(nil) // trigger cleanup cleanup <- time.Now() for { tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks() if len(tasks) == 0 { break } time.Sleep(5 * time.Millisecond) } // Getting here without deadlocking is a pass }
func TestBatchContainerHappyPath(t *testing.T) { ctrl, client, taskEngine := mocks(t, &defaultConfig) defer ctrl.Finish() ttime.SetTime(dte_test_time) sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) eventsReported := sync.WaitGroup{} dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerCreated) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerRunning) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } eventsReported.Wait() // Wait for all events to be consumed prior to moving it towards stopped; we // don't want to race the below with these or we'll end up with the "going // backwards in state" stop and we haven't 'expect'd for that exitCode := 0 // And then docker reports that sleep died, as sleep is wont to do eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}} if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") if *cont.ExitCode != 0 { t.Fatal("Exit code should be present") } } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } // Extra events should not block forever; duplicate acs and docker events are possible go func() { eventStream <- dockerEvent(api.ContainerStopped) }() go func() { eventStream <- dockerEvent(api.ContainerStopped) }() sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.DesiredStatus = api.TaskStopped taskEngine.AddTask(sleepTaskStop) // As above, duplicate events should not be a problem taskEngine.AddTask(sleepTaskStop) taskEngine.AddTask(sleepTaskStop) // Expect a bunch of steady state 'poll' describes when we warp 4 hours client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer("containerId").Return(nil) dte_test_time.Warp(4 * time.Hour) go func() { eventStream <- dockerEvent(api.ContainerStopped) }() // Wait for the task to actually be dead; if we just fallthrough immediately, // the remove might not have happened (expectation failure) for { tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks() if len(tasks) == 0 { break } time.Sleep(5 * time.Millisecond) } }
func TestSteadyStatePoll(t *testing.T) { ctrl, client, taskEngine := mocks(t, &config.Config{}) defer ctrl.Finish() ttime.SetTime(dte_test_time) sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { go func() { eventStream <- dockerEvent(api.ContainerRunning) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } // Two steady state oks, one stop gomock.InOrder( client.EXPECT().DescribeContainer("containerId").Return(api.ContainerRunning, DockerContainerMetadata{DockerId: "containerId"}).Times(2), client.EXPECT().DescribeContainer("containerId").Return(api.ContainerStopped, DockerContainerMetadata{DockerId: "containerId"}), ) // Due to how the mock time works, we actually have to warp 10 minutes per // steady-state event. That's 10 per timeout + 10 per describe * 1 container. // The reason for this is the '.After' call happens regardless of whether the // value gets read, and the test sleep will add that time to elapsed, even // though in real-time-units that much time would not have elapsed. dte_test_time.Warp(60 * time.Minute) contEvent := <-contEvents if contEvent.Status != api.ContainerStopped { t.Error("Expected container to be stopped") } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } }
func TestStartTimeoutThenStart(t *testing.T) { ctrl, client, taskEngine := mocks(t, &defaultConfig) defer ctrl.Finish() ttime.SetTime(dte_test_time) sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { meta := DockerContainerMetadata{ DockerId: "containerId", } return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}}) // Expect it to try to stop the container before going on; // in the future the agent might optimize to not stop unless the known // status is running, at which poitn this can be safeuly removed client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: errors.New("Cannot start")}) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) // Expect it to go to stopped contEvent := <-contEvents if contEvent.Status != api.ContainerStopped { t.Fatal("Expected container to timeout on start and stop") } *contEvent.SentStatus = api.ContainerStopped taskEvent := <-taskEvents if taskEvent.Status != api.TaskStopped { t.Fatal("And then task") } *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } // Expect it to try to stop it once now client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: errors.New("Cannot start")}) // Now surprise surprise, it actually did start! eventStream <- dockerEvent(api.ContainerRunning) // However, if it starts again, we should not see it be killed; no additional expect eventStream <- dockerEvent(api.ContainerRunning) eventStream <- dockerEvent(api.ContainerRunning) select { case <-taskEvents: t.Fatal("Should be out of events") case ev := <-contEvents: t.Fatal("Should be out of events", ev) default: } }
func TestBatchContainerHappyPath(t *testing.T) { ctrl, client, taskEngine := mocks(t, &config.Config{}) defer ctrl.Finish() ttime.SetTime(test_time) sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan engine.DockerContainerChangeEvent) dockerEvent := func(status api.ContainerStatus) engine.DockerContainerChangeEvent { meta := engine.DockerContainerMetadata{ DockerId: "containerId", } return engine.DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} } client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { client.EXPECT().PullImage(container.Image).Return(engine.DockerContainerMetadata{}) dockerConfig, err := sleepTask.DockerConfig(container) if err != nil { t.Fatal(err) } client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) { go func() { eventStream <- dockerEvent(api.ContainerCreated) }() }).Return(engine.DockerContainerMetadata{DockerId: "containerId"}) client.EXPECT().StartContainer("containerId").Do(func(id string) { go func() { eventStream <- dockerEvent(api.ContainerRunning) }() }).Return(engine.DockerContainerMetadata{DockerId: "containerId"}) } err := taskEngine.Init() taskEvents, contEvents := taskEngine.TaskEvents() if err != nil { t.Fatal(err) } taskEngine.AddTask(sleepTask) if (<-contEvents).Status != api.ContainerRunning { t.Fatal("Expected container to run first") } if (<-taskEvents).Status != api.TaskRunning { t.Fatal("And then task") } select { case <-taskEvents: t.Fatal("Should be out of events") case <-contEvents: t.Fatal("Should be out of events") default: } exitCode := 0 // And then docker reports that sleep died, as sleep is wont to do eventStream <- engine.DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: engine.DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}} if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") if *cont.ExitCode != 0 { t.Fatal("Exit code should be present") } } if (<-taskEvents).Status != api.TaskStopped { t.Fatal("And then task") } // Extra events should not block forever; duplicate acs and docker events are possible go func() { eventStream <- dockerEvent(api.ContainerStopped) }() go func() { eventStream <- dockerEvent(api.ContainerStopped) }() sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.DesiredStatus = api.TaskStopped taskEngine.AddTask(sleepTaskStop) // As above, duplicate events should not be a problem taskEngine.AddTask(sleepTaskStop) taskEngine.AddTask(sleepTaskStop) // Expect a bunch of steady state 'poll' describes when we warp 4 hours client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer("containerId").Return(nil) test_time.Warp(4 * time.Hour) go func() { eventStream <- dockerEvent(api.ContainerStopped) }() for { tasks, _ := taskEngine.(*engine.DockerTaskEngine).ListTasks() if len(tasks) == 0 { break } time.Sleep(5 * time.Millisecond) } }