func TestRetryWithBackoff(t *testing.T) { test_time := ttime.NewTestTime() test_time.LudicrousSpeed(true) ttime.SetTime(test_time) start := ttime.Now() counter := 3 RetryWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error { if counter == 0 { return nil } counter-- return errors.New("err") }) if counter != 0 { t.Error("Counter didn't go to 0; didn't get retried enough") } testTime := ttime.Since(start) if testTime.Seconds() < .29 || testTime.Seconds() > .31 { t.Error("Retry didn't backoff for as long as expected") } start = ttime.Now() RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error { return NewRetriableError(NewRetriable(false), errors.New("can't retry")) }) if ttime.Since(start).Seconds() > .1 { t.Error("Retry for the trivial function took too long") } }
func TestServerExceptionRetries(t *testing.T) { ctrl, client, mockRoundTripper := setup(t) defer ctrl.Finish() timesCalled := 0 // This resp.Body song and dance is because it *must* be reset between // retries for the sdk to behave sanely; it rewinds request bodies, not // response bodies. The actual server would, indeed put a new body each time // so this is not a bad thing to do resp := operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`) mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("DiscoverPollEndpoint"))).AnyTimes().Do(func(_ interface{}) { timesCalled++ resp.Body = operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`).Body }).Return(resp, nil).AnyTimes() start := ttime.Now() _, err := client.DiscoverPollEndpoint("foo") if err == nil { t.Error("Expected it to error after retrying") } duration := ttime.Since(start) if duration < 100*time.Millisecond { t.Error("Retries should have taken some time; took " + duration.String()) } if timesCalled < 2 || timesCalled > 10 { // Actaully 4 at the time of writing, but a reasonable range is fine t.Error("Retries should happen a reasonable number of times") } }
func TestRetryNWithBackoff(t *testing.T) { test_time := ttime.NewTestTime() test_time.LudicrousSpeed(true) ttime.SetTime(test_time) start := ttime.Now() counter := 3 err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error { counter-- return errors.New("err") }) if counter != 1 { t.Error("Should have stopped after two tries") } if err == nil { t.Error("Should have returned appropriate error") } testTime := ttime.Since(start) // Expect that it tried twice, sleeping once between them if testTime.Seconds() < 0.09 || testTime.Seconds() > 0.11 { t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds()) } start = ttime.Now() counter = 3 err = RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error { counter-- if counter == 0 { return nil } return errors.New("err") }) testTime = ttime.Since(start) if counter != 0 { t.Errorf("Counter expected to be 0, was %v", counter) } if err != nil { t.Errorf("Expected no error, got %v", err) } // 3 tries; 2 backoffs if testTime.Seconds() < 0.190 || testTime.Seconds() > 0.210 { t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds()) } }
func TestSubmitRetries(t *testing.T) { ctrl, client, mockRoundTripper := setup(t) defer ctrl.Finish() timesCalled := 0 resp := operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`) mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("SubmitContainerStateChange"))).AnyTimes().Do(func(_ interface{}) { timesCalled++ resp.Body = operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`).Body }).Return(resp, nil) start := ttime.Now() err := client.SubmitContainerStateChange(ContainerStateChange{ContainerName: "foo", TaskArn: "bar", Status: ContainerRunning}) if err == nil { t.Fatal("Expected it to error after retrying") } duration := ttime.Since(start) if duration < 23*time.Hour || duration > 25*time.Hour { t.Fatal("Retries should have taken roughly 24 hours; took " + duration.String()) } if timesCalled < 10 { t.Fatal("Expected to be called many times") } }
func TestDockerStopTimeout(t *testing.T) { os.Setenv("ECS_CONTAINER_STOP_TIMEOUT", testDockerStopTimeout.String()) defer os.Unsetenv("ECS_CONTAINER_STOP_TIMEOUT") cfg := defaultTestConfig() taskEngine, done, _ := setup(cfg, t) dockerTaskEngine := taskEngine.(*DockerTaskEngine) if dockerTaskEngine.cfg.DockerStopTimeout != testDockerStopTimeout { t.Errorf("Expect the docker stop timeout read from environment variable when ECS_CONTAINER_STOP_TIMEOUT is set, %v", dockerTaskEngine.cfg.DockerStopTimeout) } testTask := createTestTask("TestDockerStopTimeout") testTask.Containers = append(testTask.Containers, createTestContainer()) testTask.Containers[0].Command = []string{"sh", "-c", "while true; do echo `date +%T`; sleep 1s; done;"} testTask.Containers[0].Image = testBusyboxImage testTask.Containers[0].Name = "test-docker-timeout" taskEvents, contEvents := dockerTaskEngine.TaskEvents() ctx, cancel := context.WithCancel(context.Background()) go func() { for { select { case <-taskEvents: case <-ctx.Done(): return } } }() defer func() { done() cancel() }() go dockerTaskEngine.AddTask(testTask) for contEvent := range contEvents { if contEvent.TaskArn != testTask.Arn { continue } if contEvent.Status == api.ContainerRunning { break } if contEvent.Status > api.ContainerRunning { t.Error("Expect container to run not stop") } } startTime := ttime.Now() dockerTaskEngine.stopContainer(testTask, testTask.Containers[0]) for contEvent := range contEvents { if contEvent.TaskArn != testTask.Arn { continue } if contEvent.Status == api.ContainerRunning { break } if contEvent.Status > api.ContainerStopped { t.Error("Expect container to stop") } } if ttime.Since(startTime) < testDockerStopTimeout { t.Errorf("Container stopped before the timeout: %v", ttime.Since(startTime)) } if ttime.Since(startTime) > testDockerStopTimeout+1*time.Second { t.Errorf("Container should have stopped eariler, but stopped after %v", ttime.Since(startTime)) } }