func TestStopWithPendingStops(t *testing.T) {
ctrl, client, testTime, taskEngine, _, imageManager := mocks(t, &defaultConfig)
defer ctrl.Finish()
testTime.EXPECT().Now().AnyTimes()
testTime.EXPECT().After(gomock.Any()).AnyTimes()
sleepTask1 := testdata.LoadTask("sleep5")
sleepTask1.StartSequenceNumber = 5
sleepTask2 := testdata.LoadTask("sleep5")
sleepTask2.Arn = "arn2"
eventStream := make(chan DockerContainerChangeEvent)
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
err := taskEngine.Init()
if err != nil {
t.Fatal(err)
}
taskEvents, contEvents := taskEngine.TaskEvents()
go func() {
for {
<-taskEvents
}
}()
go func() {
for {
<-contEvents
}
}()
pullDone := make(chan bool)
pullInvoked := make(chan bool)
client.EXPECT().PullImage(gomock.Any(), nil).Do(func(x, y interface{}) {
pullInvoked <- true
<-pullDone
})
imageManager.EXPECT().AddContainerReferenceToImageState(gomock.Any()).AnyTimes()
imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).AnyTimes()
taskEngine.AddTask(sleepTask2)
<-pullInvoked
stopSleep2 := *sleepTask2
stopSleep2.SetDesiredStatus(api.TaskStopped)
stopSleep2.StopSequenceNumber = 4
taskEngine.AddTask(&stopSleep2)
taskEngine.AddTask(sleepTask1)
stopSleep1 := *sleepTask1
stopSleep1.SetDesiredStatus(api.TaskStopped)
stopSleep1.StopSequenceNumber = 5
taskEngine.AddTask(&stopSleep1)
pullDone <- true
// this means the PullImage is only called once due to the task is stopped before it
// gets the pull image lock
}
func TestStopWithPendingStops(t *testing.T) {
ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
testTime.EXPECT().Now().AnyTimes()
testTime.EXPECT().After(gomock.Any()).AnyTimes()
sleepTask1 := testdata.LoadTask("sleep5")
sleepTask1.StartSequenceNumber = 5
sleepTask2 := testdata.LoadTask("sleep5")
sleepTask2.Arn = "arn2"
eventStream := make(chan DockerContainerChangeEvent)
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
err := taskEngine.Init()
if err != nil {
t.Fatal(err)
}
taskEvents, contEvents := taskEngine.TaskEvents()
go func() {
for {
<-taskEvents
}
}()
go func() {
for {
<-contEvents
}
}()
pulling := make(chan bool)
client.EXPECT().PullImage(gomock.Any(), nil).Do(func(x, y interface{}) {
<-pulling
})
taskEngine.AddTask(sleepTask2)
stopSleep2 := *sleepTask2
stopSleep2.DesiredStatus = api.TaskStopped
stopSleep2.StopSequenceNumber = 4
taskEngine.AddTask(&stopSleep2)
taskEngine.AddTask(sleepTask1)
stopSleep1 := *sleepTask1
stopSleep1.DesiredStatus = api.TaskStopped
stopSleep1.StopSequenceNumber = 5
taskEngine.AddTask(&stopSleep1)
pulling <- true
// If we get here without deadlocking, we passed the test
}
func TestGetTaskByArn(t *testing.T) {
// Need a mock client as AddTask not only adds a task to the engine, but
// also causes the engine to progress the task.
ctrl, client, _, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
eventStream := make(chan DockerContainerChangeEvent)
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
client.EXPECT().PullImage(gomock.Any(), gomock.Any()).AnyTimes() // TODO change to MaxTimes(1)
err := taskEngine.Init()
if err != nil {
t.Fatal(err)
}
defer taskEngine.Disable()
sleepTask := testdata.LoadTask("sleep5")
sleepTaskArn := sleepTask.Arn
taskEngine.AddTask(sleepTask)
_, found := taskEngine.GetTaskByArn(sleepTaskArn)
if !found {
t.Fatalf("Task %s not found", sleepTaskArn)
}
_, found = taskEngine.GetTaskByArn(sleepTaskArn + "arn")
if found {
t.Fatal("Task with invalid arn found in the task engine")
}
}
func TestCreateContainerForceSave(t *testing.T) {
ctrl, client, _, privateTaskEngine, _ := mocks(t, &config.Config{})
saver := mock_statemanager.NewMockStateManager(ctrl)
defer ctrl.Finish()
taskEngine, _ := privateTaskEngine.(*DockerTaskEngine)
taskEngine.SetSaver(saver)
sleepTask := testdata.LoadTask("sleep5")
sleepContainer, _ := sleepTask.ContainerByName("sleep5")
gomock.InOrder(
saver.EXPECT().ForceSave().Do(func() interface{} {
task, ok := taskEngine.state.TaskByArn(sleepTask.Arn)
if task == nil || !ok {
t.Fatalf("Expected task with arn %s", sleepTask.Arn)
}
_, ok = task.ContainerByName("sleep5")
if !ok {
t.Error("Expected container sleep5")
}
return nil
}),
client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()),
)
metadata := taskEngine.createContainer(sleepTask, sleepContainer)
if metadata.Error != nil {
t.Error("Unexpected error", metadata.Error)
}
}
// TestTaskTransitionWhenStopContainerTimesout tests that task transitions to stopped
// only when terminal events are recieved from docker event stream when
// StopContainer times out
func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) {
ctrl, client, mockTime, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
mockTime.EXPECT().After(gomock.Any()).AnyTimes()
containerStopTimeoutError := DockerContainerMetadata{Error: &DockerTimeoutError{transition: "stop", duration: stopContainerTimeout}}
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
// Container config should get updated with this during CreateContainer
dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn
dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family
dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version
dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
// StartContainer returns timeout error. This should cause the engine
// to transition the task to STOPPED and to stop all containers of
// the task
client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}})
gomock.InOrder(
// StopContainer times out as well
client.EXPECT().StopContainer("containerId").Do(func(id string) {
// Simulate docker actually stopping the container even though
// StopContainer in container engine times out
go func() {
eventStream <- dockerEvent(api.ContainerStopped)
}()
}).Return(containerStopTimeoutError),
// Since task is not in steady state, progressContainers causes
// another invocation of StopContainer. Return a timeout error
// for that as well
// TODO change AnyTimes() to MinTimes(1) after updating gomock
client.EXPECT().StopContainer("containerId").Return(containerStopTimeoutError).AnyTimes(),
)
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatalf("Error getting event streams from engine: %v", err)
}
taskEngine.AddTask(sleepTask)
// Expect it to go to stopped
contEvent := <-contEvents
if contEvent.Status != api.ContainerStopped {
t.Errorf("Expected container to timeout on start and stop, got: %v", contEvent)
}
*contEvent.SentStatus = api.ContainerStopped
taskEvent := <-taskEvents
if taskEvent.Status != api.TaskStopped {
t.Errorf("Expected task to be stopped, got: %v", taskEvent)
}
*taskEvent.SentStatus = api.TaskStopped
select {
case <-taskEvents:
t.Error("Should be out of events")
case <-contEvents:
t.Error("Should be out of events")
default:
}
}
func TestBatchContainerHappyPath(t *testing.T) {
ctrl, client, mockTime, taskEngine, credentialsManager := mocks(t, &defaultConfig)
defer ctrl.Finish()
roleCredentials := &credentials.TaskIAMRoleCredentials{
IAMRoleCredentials: credentials.IAMRoleCredentials{CredentialsId: "credsid"},
}
credentialsManager.EXPECT().GetTaskCredentials(credentialsId).Return(roleCredentials, true).AnyTimes()
credentialsManager.EXPECT().RemoveCredentials(credentialsId)
sleepTask := testdata.LoadTask("sleep5")
sleepTask.SetCredentialsId(credentialsId)
eventStream := make(chan DockerContainerChangeEvent)
eventsReported := sync.WaitGroup{}
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
var createdContainerName string
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
// Container config should get updated with this during PostUnmarshalTask
credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI()
dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue)
if err != nil {
t.Fatal(err)
}
// Container config should get updated with this during CreateContainer
dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn
dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family
dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version
dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(config *docker.Config, y interface{}, containerName string) {
if !reflect.DeepEqual(dockerConfig, config) {
t.Errorf("Mismatch in container config; expected: %v, got: %v", dockerConfig, config)
}
// sleep5 task contains only one container. Just assign
// the containerName to createdContainerName
createdContainerName = containerName
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerCreated)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerRunning)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
}
steadyStateVerify := make(chan time.Time, 1)
cleanup := make(chan time.Time, 1)
mockTime.EXPECT().Now().Do(func() time.Time { return time.Now() }).AnyTimes()
mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes()
mockTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes()
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
eventsReported.Wait()
// Wait for all events to be consumed prior to moving it towards stopped; we
// don't want to race the below with these or we'll end up with the "going
// backwards in state" stop and we haven't 'expect'd for that
exitCode := 0
// And then docker reports that sleep died, as sleep is wont to do
eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}}
steadyStateVerify <- time.Now()
if cont := <-contEvents; cont.Status != api.ContainerStopped {
t.Fatal("Expected container to stop first")
if *cont.ExitCode != 0 {
t.Fatal("Exit code should be present")
}
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
// Extra events should not block forever; duplicate acs and docker events are possible
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
sleepTaskStop := testdata.LoadTask("sleep5")
sleepTaskStop.SetCredentialsId(credentialsId)
sleepTaskStop.DesiredStatus = api.TaskStopped
taskEngine.AddTask(sleepTaskStop)
// As above, duplicate events should not be a problem
taskEngine.AddTask(sleepTaskStop)
taskEngine.AddTask(sleepTaskStop)
// Expect a bunch of steady state 'poll' describes when we trigger cleanup
client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes()
client.EXPECT().RemoveContainer(gomock.Any()).Do(func(removedContainerName string) {
if createdContainerName != removedContainerName {
t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName)
}
}).Return(nil)
// trigger cleanup
cleanup <- time.Now()
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
// Wait for the task to actually be dead; if we just fallthrough immediately,
// the remove might not have happened (expectation failure)
for {
tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks()
if len(tasks) == 0 {
break
}
time.Sleep(5 * time.Millisecond)
}
}
func TestSteadyStatePoll(t *testing.T) {
ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
// Container config should get updated with this during CreateContainer
dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn
dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family
dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version
dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
go func() { eventStream <- dockerEvent(api.ContainerRunning) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
}
steadyStateVerify := make(chan time.Time, 10)
testTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes()
testTime.EXPECT().After(gomock.Any()).AnyTimes()
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
// Two steady state oks, one stop
gomock.InOrder(
client.EXPECT().DescribeContainer("containerId").Return(api.ContainerRunning, DockerContainerMetadata{DockerId: "containerId"}).Times(2),
// TODO change AnyTimes() to MinTimes(1) after updating gomock
client.EXPECT().DescribeContainer("containerId").Return(api.ContainerStopped, DockerContainerMetadata{DockerId: "containerId"}).AnyTimes(),
)
for i := 0; i < 10; i++ {
steadyStateVerify <- time.Now()
}
close(steadyStateVerify)
contEvent := <-contEvents
if contEvent.Status != api.ContainerStopped {
t.Error("Expected container to be stopped")
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
// cleanup expectations
testTime.EXPECT().Now().AnyTimes()
testTime.EXPECT().After(gomock.Any()).AnyTimes()
}
func TestStartTimeoutThenStart(t *testing.T) {
ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
testTime.EXPECT().After(gomock.Any())
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
// Container config should get updated with this during CreateContainer
dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn
dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name
dockerConfig.Labels["com.amazonaws.ecs.task-definition-family"] = sleepTask.Family
dockerConfig.Labels["com.amazonaws.ecs.task-definition-version"] = sleepTask.Version
dockerConfig.Labels["com.amazonaws.ecs.cluster"] = ""
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}})
// Expect it to try to stop the container before going on;
// in the future the agent might optimize to not stop unless the known
// status is running, at which point this can be safely removed
client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: CannotXContainerError{transition: "start", msg: "Cannot start"}})
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
// Expect it to go to stopped
contEvent := <-contEvents
if contEvent.Status != api.ContainerStopped {
t.Fatal("Expected container to timeout on start and stop")
}
*contEvent.SentStatus = api.ContainerStopped
taskEvent := <-taskEvents
if taskEvent.Status != api.TaskStopped {
t.Fatal("And then task")
}
*taskEvent.SentStatus = api.TaskStopped
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
// Expect it to try to stop it once now
client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: CannotXContainerError{transition: "start", msg: "Cannot start"}}).AnyTimes()
// Now surprise surprise, it actually did start!
eventStream <- dockerEvent(api.ContainerRunning)
// However, if it starts again, we should not see it be killed; no additional expect
eventStream <- dockerEvent(api.ContainerRunning)
eventStream <- dockerEvent(api.ContainerRunning)
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case ev := <-contEvents:
t.Fatal("Should be out of events", ev)
default:
}
}
func TestRemoveEvents(t *testing.T) {
ctrl, client, testTime, taskEngine, _ := mocks(t, &defaultConfig)
defer ctrl.Finish()
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
eventsReported := sync.WaitGroup{}
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
var createdContainerName string
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(x, y interface{}, containerName string) {
// sleep5 task contains only one container. Just assign
// the containerName to createdContainerName
createdContainerName = containerName
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerCreated)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerRunning)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
}
steadyStateVerify := make(chan time.Time, 1)
cleanup := make(chan time.Time, 1)
testTime.EXPECT().Now().Do(func() time.Time { return time.Now() }).AnyTimes()
testTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes()
testTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes()
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
eventsReported.Wait()
// Wait for all events to be consumed prior to moving it towards stopped; we
// don't want to race the below with these or we'll end up with the "going
// backwards in state" stop and we haven't 'expect'd for that
exitCode := 0
// And then docker reports that sleep died, as sleep is wont to do
eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}}
steadyStateVerify <- time.Now()
if cont := <-contEvents; cont.Status != api.ContainerStopped {
t.Fatal("Expected container to stop first")
if *cont.ExitCode != 0 {
t.Fatal("Exit code should be present")
}
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
sleepTaskStop := testdata.LoadTask("sleep5")
sleepTaskStop.DesiredStatus = api.TaskStopped
taskEngine.AddTask(sleepTaskStop)
// Expect a bunch of steady state 'poll' describes when we warp 4 hours
client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes()
client.EXPECT().RemoveContainer(gomock.Any()).Do(func(removedContainerName string) {
if createdContainerName != removedContainerName {
t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName)
}
// Emit a couple events for the task before the remove finishes; make sure this gets handled appropriately
eventStream <- dockerEvent(api.ContainerStopped)
eventStream <- dockerEvent(api.ContainerStopped)
}).Return(nil)
// trigger cleanup
cleanup <- time.Now()
for {
tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks()
if len(tasks) == 0 {
break
}
time.Sleep(5 * time.Millisecond)
}
// Getting here without deadlocking is a pass
}
func TestBatchContainerHappyPath(t *testing.T) {
ctrl, client, taskEngine := mocks(t, &defaultConfig)
defer ctrl.Finish()
ttime.SetTime(dte_test_time)
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
eventsReported := sync.WaitGroup{}
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerCreated)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
eventsReported.Add(1)
go func() {
eventStream <- dockerEvent(api.ContainerRunning)
eventsReported.Done()
}()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
eventsReported.Wait()
// Wait for all events to be consumed prior to moving it towards stopped; we
// don't want to race the below with these or we'll end up with the "going
// backwards in state" stop and we haven't 'expect'd for that
exitCode := 0
// And then docker reports that sleep died, as sleep is wont to do
eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}}
if cont := <-contEvents; cont.Status != api.ContainerStopped {
t.Fatal("Expected container to stop first")
if *cont.ExitCode != 0 {
t.Fatal("Exit code should be present")
}
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
// Extra events should not block forever; duplicate acs and docker events are possible
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
sleepTaskStop := testdata.LoadTask("sleep5")
sleepTaskStop.DesiredStatus = api.TaskStopped
taskEngine.AddTask(sleepTaskStop)
// As above, duplicate events should not be a problem
taskEngine.AddTask(sleepTaskStop)
taskEngine.AddTask(sleepTaskStop)
// Expect a bunch of steady state 'poll' describes when we warp 4 hours
client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes()
client.EXPECT().RemoveContainer("containerId").Return(nil)
dte_test_time.Warp(4 * time.Hour)
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
// Wait for the task to actually be dead; if we just fallthrough immediately,
// the remove might not have happened (expectation failure)
for {
tasks, _ := taskEngine.(*DockerTaskEngine).ListTasks()
if len(tasks) == 0 {
break
}
time.Sleep(5 * time.Millisecond)
}
}
func TestSteadyStatePoll(t *testing.T) {
ctrl, client, taskEngine := mocks(t, &config.Config{})
defer ctrl.Finish()
ttime.SetTime(dte_test_time)
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
go func() { eventStream <- dockerEvent(api.ContainerRunning) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
// Two steady state oks, one stop
gomock.InOrder(
client.EXPECT().DescribeContainer("containerId").Return(api.ContainerRunning, DockerContainerMetadata{DockerId: "containerId"}).Times(2),
client.EXPECT().DescribeContainer("containerId").Return(api.ContainerStopped, DockerContainerMetadata{DockerId: "containerId"}),
)
// Due to how the mock time works, we actually have to warp 10 minutes per
// steady-state event. That's 10 per timeout + 10 per describe * 1 container.
// The reason for this is the '.After' call happens regardless of whether the
// value gets read, and the test sleep will add that time to elapsed, even
// though in real-time-units that much time would not have elapsed.
dte_test_time.Warp(60 * time.Minute)
contEvent := <-contEvents
if contEvent.Status != api.ContainerStopped {
t.Error("Expected container to be stopped")
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
}
func TestStartTimeoutThenStart(t *testing.T) {
ctrl, client, taskEngine := mocks(t, &defaultConfig)
defer ctrl.Finish()
ttime.SetTime(dte_test_time)
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan DockerContainerChangeEvent)
dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent {
meta := DockerContainerMetadata{
DockerId: "containerId",
}
return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image, nil).Return(DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Return(DockerContainerMetadata{Error: &DockerTimeoutError{}})
// Expect it to try to stop the container before going on;
// in the future the agent might optimize to not stop unless the known
// status is running, at which poitn this can be safeuly removed
client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: errors.New("Cannot start")})
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
// Expect it to go to stopped
contEvent := <-contEvents
if contEvent.Status != api.ContainerStopped {
t.Fatal("Expected container to timeout on start and stop")
}
*contEvent.SentStatus = api.ContainerStopped
taskEvent := <-taskEvents
if taskEvent.Status != api.TaskStopped {
t.Fatal("And then task")
}
*taskEvent.SentStatus = api.TaskStopped
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
// Expect it to try to stop it once now
client.EXPECT().StopContainer("containerId").Return(DockerContainerMetadata{Error: errors.New("Cannot start")})
// Now surprise surprise, it actually did start!
eventStream <- dockerEvent(api.ContainerRunning)
// However, if it starts again, we should not see it be killed; no additional expect
eventStream <- dockerEvent(api.ContainerRunning)
eventStream <- dockerEvent(api.ContainerRunning)
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case ev := <-contEvents:
t.Fatal("Should be out of events", ev)
default:
}
}
func TestBatchContainerHappyPath(t *testing.T) {
ctrl, client, taskEngine := mocks(t, &config.Config{})
defer ctrl.Finish()
ttime.SetTime(test_time)
sleepTask := testdata.LoadTask("sleep5")
eventStream := make(chan engine.DockerContainerChangeEvent)
dockerEvent := func(status api.ContainerStatus) engine.DockerContainerChangeEvent {
meta := engine.DockerContainerMetadata{
DockerId: "containerId",
}
return engine.DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta}
}
client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil)
for _, container := range sleepTask.Containers {
client.EXPECT().PullImage(container.Image).Return(engine.DockerContainerMetadata{})
dockerConfig, err := sleepTask.DockerConfig(container)
if err != nil {
t.Fatal(err)
}
client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any()).Do(func(x, y, z interface{}) {
go func() { eventStream <- dockerEvent(api.ContainerCreated) }()
}).Return(engine.DockerContainerMetadata{DockerId: "containerId"})
client.EXPECT().StartContainer("containerId").Do(func(id string) {
go func() { eventStream <- dockerEvent(api.ContainerRunning) }()
}).Return(engine.DockerContainerMetadata{DockerId: "containerId"})
}
err := taskEngine.Init()
taskEvents, contEvents := taskEngine.TaskEvents()
if err != nil {
t.Fatal(err)
}
taskEngine.AddTask(sleepTask)
if (<-contEvents).Status != api.ContainerRunning {
t.Fatal("Expected container to run first")
}
if (<-taskEvents).Status != api.TaskRunning {
t.Fatal("And then task")
}
select {
case <-taskEvents:
t.Fatal("Should be out of events")
case <-contEvents:
t.Fatal("Should be out of events")
default:
}
exitCode := 0
// And then docker reports that sleep died, as sleep is wont to do
eventStream <- engine.DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: engine.DockerContainerMetadata{DockerId: "containerId", ExitCode: &exitCode}}
if cont := <-contEvents; cont.Status != api.ContainerStopped {
t.Fatal("Expected container to stop first")
if *cont.ExitCode != 0 {
t.Fatal("Exit code should be present")
}
}
if (<-taskEvents).Status != api.TaskStopped {
t.Fatal("And then task")
}
// Extra events should not block forever; duplicate acs and docker events are possible
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
sleepTaskStop := testdata.LoadTask("sleep5")
sleepTaskStop.DesiredStatus = api.TaskStopped
taskEngine.AddTask(sleepTaskStop)
// As above, duplicate events should not be a problem
taskEngine.AddTask(sleepTaskStop)
taskEngine.AddTask(sleepTaskStop)
// Expect a bunch of steady state 'poll' describes when we warp 4 hours
client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes()
client.EXPECT().RemoveContainer("containerId").Return(nil)
test_time.Warp(4 * time.Hour)
go func() { eventStream <- dockerEvent(api.ContainerStopped) }()
for {
tasks, _ := taskEngine.(*engine.DockerTaskEngine).ListTasks()
if len(tasks) == 0 {
break
}
time.Sleep(5 * time.Millisecond)
}
}