func TestRetryWithBackoff(t *testing.T) {
test_time := ttime.NewTestTime()
test_time.LudicrousSpeed(true)
ttime.SetTime(test_time)
start := ttime.Now()
counter := 3
RetryWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error {
if counter == 0 {
return nil
}
counter--
return errors.New("err")
})
if counter != 0 {
t.Error("Counter didn't go to 0; didn't get retried enough")
}
testTime := ttime.Since(start)
if testTime.Seconds() < .29 || testTime.Seconds() > .31 {
t.Error("Retry didn't backoff for as long as expected")
}
start = ttime.Now()
RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error {
return NewRetriableError(NewRetriable(false), errors.New("can't retry"))
})
if ttime.Since(start).Seconds() > .1 {
t.Error("Retry for the trivial function took too long")
}
}
func TestServerExceptionRetries(t *testing.T) {
ctrl, client, mockRoundTripper := setup(t)
defer ctrl.Finish()
timesCalled := 0
// This resp.Body song and dance is because it *must* be reset between
// retries for the sdk to behave sanely; it rewinds request bodies, not
// response bodies. The actual server would, indeed put a new body each time
// so this is not a bad thing to do
resp := operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`)
mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("DiscoverPollEndpoint"))).AnyTimes().Do(func(_ interface{}) {
timesCalled++
resp.Body = operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`).Body
}).Return(resp, nil).AnyTimes()
start := ttime.Now()
_, err := client.DiscoverPollEndpoint("foo")
if err == nil {
t.Error("Expected it to error after retrying")
}
duration := ttime.Since(start)
if duration < 100*time.Millisecond {
t.Error("Retries should have taken some time; took " + duration.String())
}
if timesCalled < 2 || timesCalled > 10 {
// Actaully 4 at the time of writing, but a reasonable range is fine
t.Error("Retries should happen a reasonable number of times")
}
}
func TestRetryNWithBackoff(t *testing.T) {
test_time := ttime.NewTestTime()
test_time.LudicrousSpeed(true)
ttime.SetTime(test_time)
start := ttime.Now()
counter := 3
err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error {
counter--
return errors.New("err")
})
if counter != 1 {
t.Error("Should have stopped after two tries")
}
if err == nil {
t.Error("Should have returned appropriate error")
}
testTime := ttime.Since(start)
// Expect that it tried twice, sleeping once between them
if testTime.Seconds() < 0.09 || testTime.Seconds() > 0.11 {
t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
}
start = ttime.Now()
counter = 3
err = RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error {
counter--
if counter == 0 {
return nil
}
return errors.New("err")
})
testTime = ttime.Since(start)
if counter != 0 {
t.Errorf("Counter expected to be 0, was %v", counter)
}
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
// 3 tries; 2 backoffs
if testTime.Seconds() < 0.190 || testTime.Seconds() > 0.210 {
t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
}
}
func TestSubmitRetries(t *testing.T) {
ctrl, client, mockRoundTripper := setup(t)
defer ctrl.Finish()
timesCalled := 0
resp := operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`)
mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("SubmitContainerStateChange"))).AnyTimes().Do(func(_ interface{}) {
timesCalled++
resp.Body = operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`).Body
}).Return(resp, nil)
start := ttime.Now()
err := client.SubmitContainerStateChange(ContainerStateChange{ContainerName: "foo", TaskArn: "bar", Status: ContainerRunning})
if err == nil {
t.Fatal("Expected it to error after retrying")
}
duration := ttime.Since(start)
if duration < 23*time.Hour || duration > 25*time.Hour {
t.Fatal("Retries should have taken roughly 24 hours; took " + duration.String())
}
if timesCalled < 10 {
t.Fatal("Expected to be called many times")
}
}
func TestDockerStopTimeout(t *testing.T) {
os.Setenv("ECS_CONTAINER_STOP_TIMEOUT", testDockerStopTimeout.String())
defer os.Unsetenv("ECS_CONTAINER_STOP_TIMEOUT")
cfg := defaultTestConfig()
taskEngine, done, _ := setup(cfg, t)
dockerTaskEngine := taskEngine.(*DockerTaskEngine)
if dockerTaskEngine.cfg.DockerStopTimeout != testDockerStopTimeout {
t.Errorf("Expect the docker stop timeout read from environment variable when ECS_CONTAINER_STOP_TIMEOUT is set, %v", dockerTaskEngine.cfg.DockerStopTimeout)
}
testTask := createTestTask("TestDockerStopTimeout")
testTask.Containers = append(testTask.Containers, createTestContainer())
testTask.Containers[0].Command = []string{"sh", "-c", "while true; do echo `date +%T`; sleep 1s; done;"}
testTask.Containers[0].Image = testBusyboxImage
testTask.Containers[0].Name = "test-docker-timeout"
taskEvents, contEvents := dockerTaskEngine.TaskEvents()
ctx, cancel := context.WithCancel(context.Background())
go func() {
for {
select {
case <-taskEvents:
case <-ctx.Done():
return
}
}
}()
defer func() {
done()
cancel()
}()
go dockerTaskEngine.AddTask(testTask)
for contEvent := range contEvents {
if contEvent.TaskArn != testTask.Arn {
continue
}
if contEvent.Status == api.ContainerRunning {
break
}
if contEvent.Status > api.ContainerRunning {
t.Error("Expect container to run not stop")
}
}
startTime := ttime.Now()
dockerTaskEngine.stopContainer(testTask, testTask.Containers[0])
for contEvent := range contEvents {
if contEvent.TaskArn != testTask.Arn {
continue
}
if contEvent.Status == api.ContainerRunning {
break
}
if contEvent.Status > api.ContainerStopped {
t.Error("Expect container to stop")
}
}
if ttime.Since(startTime) < testDockerStopTimeout {
t.Errorf("Container stopped before the timeout: %v", ttime.Since(startTime))
}
if ttime.Since(startTime) > testDockerStopTimeout+1*time.Second {
t.Errorf("Container should have stopped eariler, but stopped after %v", ttime.Since(startTime))
}
}