Exemplo n.º 1
0
func TestRetryWithBackoff(t *testing.T) {
	test_time := ttime.NewTestTime()
	test_time.LudicrousSpeed(true)
	ttime.SetTime(test_time)

	start := ttime.Now()

	counter := 3
	RetryWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error {
		if counter == 0 {
			return nil
		}
		counter--
		return errors.New("err")
	})
	if counter != 0 {
		t.Error("Counter didn't go to 0; didn't get retried enough")
	}
	testTime := ttime.Since(start)

	if testTime.Seconds() < .29 || testTime.Seconds() > .31 {
		t.Error("Retry didn't backoff for as long as expected")
	}

	start = ttime.Now()
	RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error {
		return NewRetriableError(NewRetriable(false), errors.New("can't retry"))
	})

	if ttime.Since(start).Seconds() > .1 {
		t.Error("Retry for the trivial function took too long")
	}
}
Exemplo n.º 2
0
func TestServerExceptionRetries(t *testing.T) {
	ctrl, client, mockRoundTripper := setup(t)
	defer ctrl.Finish()

	timesCalled := 0
	// This resp.Body song and dance is because it *must* be reset between
	// retries for the sdk to behave sanely; it rewinds request bodies, not
	// response bodies. The actual server would, indeed put a new body each time
	// so this is not a bad thing to do
	resp := operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`)
	mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("DiscoverPollEndpoint"))).AnyTimes().Do(func(_ interface{}) {
		timesCalled++
		resp.Body = operationErrorResp(500, `{"__type":"BadStuffHappenedException","message":"something went wrong"}`).Body
	}).Return(resp, nil).AnyTimes()

	start := ttime.Now()
	_, err := client.DiscoverPollEndpoint("foo")
	if err == nil {
		t.Error("Expected it to error after retrying")
	}
	duration := ttime.Since(start)
	if duration < 100*time.Millisecond {
		t.Error("Retries should have taken some time; took " + duration.String())
	}
	if timesCalled < 2 || timesCalled > 10 {
		// Actaully 4 at the time of writing, but a reasonable range is fine
		t.Error("Retries should happen a reasonable number of times")
	}
}
Exemplo n.º 3
0
func TestRetryNWithBackoff(t *testing.T) {
	test_time := ttime.NewTestTime()
	test_time.LudicrousSpeed(true)
	ttime.SetTime(test_time)

	start := ttime.Now()

	counter := 3
	err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error {
		counter--
		return errors.New("err")
	})
	if counter != 1 {
		t.Error("Should have stopped after two tries")
	}
	if err == nil {
		t.Error("Should have returned appropriate error")
	}
	testTime := ttime.Since(start)
	// Expect that it tried twice, sleeping once between them
	if testTime.Seconds() < 0.09 || testTime.Seconds() > 0.11 {
		t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
	}

	start = ttime.Now()
	counter = 3
	err = RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error {
		counter--
		if counter == 0 {
			return nil
		}
		return errors.New("err")
	})
	testTime = ttime.Since(start)
	if counter != 0 {
		t.Errorf("Counter expected to be 0, was %v", counter)
	}
	if err != nil {
		t.Errorf("Expected no error, got %v", err)
	}
	// 3 tries; 2 backoffs
	if testTime.Seconds() < 0.190 || testTime.Seconds() > 0.210 {
		t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
	}
}
Exemplo n.º 4
0
func TestSubmitRetries(t *testing.T) {
	ctrl, client, mockRoundTripper := setup(t)
	defer ctrl.Finish()

	timesCalled := 0
	resp := operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`)
	mockRoundTripper.EXPECT().RoundTrip(mock_http.NewHTTPOperationMatcher(versionedOperation("SubmitContainerStateChange"))).AnyTimes().Do(func(_ interface{}) {
		timesCalled++
		resp.Body = operationErrorResp(500, `{"__type":"SubmitContainerStateChangeException","message":"something broke horribly"}`).Body
	}).Return(resp, nil)

	start := ttime.Now()
	err := client.SubmitContainerStateChange(ContainerStateChange{ContainerName: "foo", TaskArn: "bar", Status: ContainerRunning})
	if err == nil {
		t.Fatal("Expected it to error after retrying")
	}
	duration := ttime.Since(start)
	if duration < 23*time.Hour || duration > 25*time.Hour {
		t.Fatal("Retries should have taken roughly 24 hours; took " + duration.String())
	}
	if timesCalled < 10 {
		t.Fatal("Expected to be called many times")
	}
}
func TestDockerStopTimeout(t *testing.T) {
	os.Setenv("ECS_CONTAINER_STOP_TIMEOUT", testDockerStopTimeout.String())
	defer os.Unsetenv("ECS_CONTAINER_STOP_TIMEOUT")
	cfg := defaultTestConfig()

	taskEngine, done, _ := setup(cfg, t)

	dockerTaskEngine := taskEngine.(*DockerTaskEngine)

	if dockerTaskEngine.cfg.DockerStopTimeout != testDockerStopTimeout {
		t.Errorf("Expect the docker stop timeout read from environment variable when ECS_CONTAINER_STOP_TIMEOUT is set, %v", dockerTaskEngine.cfg.DockerStopTimeout)
	}
	testTask := createTestTask("TestDockerStopTimeout")
	testTask.Containers = append(testTask.Containers, createTestContainer())
	testTask.Containers[0].Command = []string{"sh", "-c", "while true; do echo `date +%T`; sleep 1s; done;"}
	testTask.Containers[0].Image = testBusyboxImage
	testTask.Containers[0].Name = "test-docker-timeout"

	taskEvents, contEvents := dockerTaskEngine.TaskEvents()
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		for {
			select {
			case <-taskEvents:
			case <-ctx.Done():
				return
			}
		}
	}()
	defer func() {
		done()
		cancel()
	}()

	go dockerTaskEngine.AddTask(testTask)

	for contEvent := range contEvents {
		if contEvent.TaskArn != testTask.Arn {
			continue
		}
		if contEvent.Status == api.ContainerRunning {
			break
		}
		if contEvent.Status > api.ContainerRunning {
			t.Error("Expect container to run not stop")
		}
	}

	startTime := ttime.Now()
	dockerTaskEngine.stopContainer(testTask, testTask.Containers[0])
	for contEvent := range contEvents {
		if contEvent.TaskArn != testTask.Arn {
			continue
		}
		if contEvent.Status == api.ContainerRunning {
			break
		}
		if contEvent.Status > api.ContainerStopped {
			t.Error("Expect container to stop")
		}
	}
	if ttime.Since(startTime) < testDockerStopTimeout {
		t.Errorf("Container stopped before the timeout: %v", ttime.Since(startTime))
	}
	if ttime.Since(startTime) > testDockerStopTimeout+1*time.Second {
		t.Errorf("Container should have stopped eariler, but stopped after %v", ttime.Since(startTime))
	}
}