func TestRetryWithBackoff(t *testing.T) {
test_time := ttime.NewTestTime()
test_time.LudicrousSpeed(true)
ttime.SetTime(test_time)
start := ttime.Now()
counter := 3
RetryWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error {
if counter == 0 {
return nil
}
counter--
return errors.New("err")
})
if counter != 0 {
t.Error("Counter didn't go to 0; didn't get retried enough")
}
testTime := ttime.Since(start)
if testTime.Seconds() < .29 || testTime.Seconds() > .31 {
t.Error("Retry didn't backoff for as long as expected")
}
start = ttime.Now()
RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error {
return NewRetriableError(NewRetriable(false), errors.New("can't retry"))
})
if ttime.Since(start).Seconds() > .1 {
t.Error("Retry for the trivial function took too long")
}
}
func TestPullImageECRSuccess(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockDocker := mock_dockeriface.NewMockClient(ctrl)
mockDocker.EXPECT().Ping().AnyTimes().Return(nil)
factory := mock_dockerclient.NewMockFactory(ctrl)
factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil)
client, _ := NewDockerGoClient(factory, "", config.NewSensitiveRawMessage([]byte{}), false)
goClient, _ := client.(*dockerGoClient)
ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl)
ecrClient := mock_ecr.NewMockECRSDK(ctrl)
goClient.ecrClientFactory = ecrClientFactory
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
registryId := "123456789012"
region := "eu-west-1"
endpointOverride := "my.endpoint"
authData := &api.RegistryAuthenticationData{
Type: "ecr",
ECRAuthData: &api.ECRAuthData{
RegistryId: registryId,
Region: region,
EndpointOverride: endpointOverride,
},
}
imageEndpoint := "registry.endpoint"
image := imageEndpoint + "/myimage:tag"
username := "******"
password := "******"
dockerAuthConfiguration := docker.AuthConfiguration{
Username: username,
Password: password,
ServerAddress: "https://" + imageEndpoint,
}
getAuthorizationTokenInput := &ecrapi.GetAuthorizationTokenInput{
RegistryIds: []*string{aws.String(registryId)},
}
ecrClientFactory.EXPECT().GetClient(region, endpointOverride).Return(ecrClient)
ecrClient.EXPECT().GetAuthorizationToken(getAuthorizationTokenInput).Return(
&ecrapi.GetAuthorizationTokenOutput{
AuthorizationData: []*ecrapi.AuthorizationData{
&ecrapi.AuthorizationData{
ProxyEndpoint: aws.String("https://" + imageEndpoint),
AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte(username + ":" + password))),
},
},
}, nil)
mockDocker.EXPECT().PullImage(
&pullImageOptsMatcher{image},
dockerAuthConfiguration,
).Return(nil)
metadata := client.PullImage(image, authData)
if metadata.Error != nil {
t.Error("Expected pull to succeed")
}
}
func dockerclientSetup(t *testing.T) (*mock_dockerclient.MockClient, *DockerGoClient, *ttime.TestTime, func()) {
ctrl := gomock.NewController(t)
mockDocker := mock_dockerclient.NewMockClient(ctrl)
client := &DockerGoClient{}
client.SetGoDockerClient(mockDocker)
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
return mockDocker, client, testTime, ctrl.Finish
}
func dockerclientSetup(t *testing.T) (*mock_dockeriface.MockClient, *DockerGoClient, *ttime.TestTime, func()) {
ctrl := gomock.NewController(t)
mockDocker := mock_dockeriface.NewMockClient(ctrl)
mockDocker.EXPECT().Ping().AnyTimes().Return(nil)
factory := mock_dockerclient.NewMockFactory(ctrl)
factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil)
client, _ := NewDockerGoClient(factory)
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
return mockDocker, client, testTime, ctrl.Finish
}
func setup(t *testing.T) (*gomock.Controller, ECSClient, *mock_http.MockRoundTripper) {
ctrl := gomock.NewController(t)
mockRoundTripper := mock_http.NewMockRoundTripper(ctrl)
mockHttpClient := httpclient.New(1*time.Second, true)
mockHttpClient.Transport.(httpclient.OverridableTransport).SetTransport(mockRoundTripper)
client := NewECSClient(credentials.AnonymousCredentials, &config.Config{AWSRegion: "us-east-1"}, mockHttpClient)
testTime := ttime.NewTestTime()
testTime.LudicrousSpeed(true)
ttime.SetTime(testTime)
return ctrl, client, mockRoundTripper
}
func dockerclientSetup(t *testing.T) (*mock_dockeriface.MockClient, *dockerGoClient, *ttime.TestTime, func()) {
ctrl := gomock.NewController(t)
mockDocker := mock_dockeriface.NewMockClient(ctrl)
mockDocker.EXPECT().Ping().AnyTimes().Return(nil)
factory := mock_dockerclient.NewMockFactory(ctrl)
factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil)
client, _ := NewDockerGoClient(factory, "", config.NewSensitiveRawMessage([]byte{}), false)
goClient, _ := client.(*dockerGoClient)
ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl)
goClient.ecrClientFactory = ecrClientFactory
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
return mockDocker, goClient, testTime, ctrl.Finish
}
func TestErrorPropogatesUp(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockGetter := mock_ec2.NewMockHttpClient(ctrl)
testClient := ec2.NewEC2MetadataClient(mockGetter)
testTime := ttime.NewTestTime()
testTime.LudicrousSpeed(true)
ttime.SetTime(testTime)
mockGetter.EXPECT().Get(ec2.EC2_METADATA_SERVICE_URL+ec2.INSTANCE_IDENTITY_DOCUMENT_RESOURCE).Return(nil, errors.New("Something broke")).AnyTimes()
_, err := testClient.InstanceIdentityDocument()
if err == nil {
t.Fatal("Expected error to result")
}
}
func TestRetryNWithBackoff(t *testing.T) {
test_time := ttime.NewTestTime()
test_time.LudicrousSpeed(true)
ttime.SetTime(test_time)
start := ttime.Now()
counter := 3
err := RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 2, func() error {
counter--
return errors.New("err")
})
if counter != 1 {
t.Error("Should have stopped after two tries")
}
if err == nil {
t.Error("Should have returned appropriate error")
}
testTime := ttime.Since(start)
// Expect that it tried twice, sleeping once between them
if testTime.Seconds() < 0.09 || testTime.Seconds() > 0.11 {
t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
}
start = ttime.Now()
counter = 3
err = RetryNWithBackoff(NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), 5, func() error {
counter--
if counter == 0 {
return nil
}
return errors.New("err")
})
testTime = ttime.Since(start)
if counter != 0 {
t.Errorf("Counter expected to be 0, was %v", counter)
}
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
// 3 tries; 2 backoffs
if testTime.Seconds() < 0.190 || testTime.Seconds() > 0.210 {
t.Errorf("Retry didn't backoff for as long as expected: %v", testTime.Seconds())
}
}
func setup(t *testing.T) (TaskEngine, func(), *ttime.TestTime) {
if testing.Short() {
t.Skip("Skipping integ test in short mode")
}
if _, err := os.Stat("/var/run/docker.sock"); err != nil {
t.Skip("Docker not running")
}
if os.Getenv("ECS_SKIP_ENGINE_INTEG_TEST") != "" {
t.Skip("ECS_SKIP_ENGINE_INTEG_TEST")
}
taskEngine := NewDockerTaskEngine(cfg, false)
taskEngine.Init()
test_time := ttime.NewTestTime()
ttime.SetTime(test_time)
return taskEngine, func() {
taskEngine.Shutdown()
test_time.Cancel()
}, test_time
}
func TestPullImageECRAuthFail(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockDocker := mock_dockeriface.NewMockClient(ctrl)
mockDocker.EXPECT().Ping().AnyTimes().Return(nil)
factory := mock_dockerclient.NewMockFactory(ctrl)
factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil)
client, _ := NewDockerGoClient(factory, "", config.NewSensitiveRawMessage([]byte{}), false)
goClient, _ := client.(*dockerGoClient)
ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl)
ecrClient := mock_ecr.NewMockECRSDK(ctrl)
goClient.ecrClientFactory = ecrClientFactory
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
registryId := "123456789012"
region := "eu-west-1"
endpointOverride := "my.endpoint"
authData := &api.RegistryAuthenticationData{
Type: "ecr",
ECRAuthData: &api.ECRAuthData{
RegistryId: registryId,
Region: region,
EndpointOverride: endpointOverride,
},
}
imageEndpoint := "registry.endpoint"
image := imageEndpoint + "/myimage:tag"
ecrClientFactory.EXPECT().GetClient(region, endpointOverride).Return(ecrClient)
ecrClient.EXPECT().GetAuthorizationToken(gomock.Any()).Return(&ecrapi.GetAuthorizationTokenOutput{}, errors.New("test error"))
metadata := client.PullImage(image, authData)
if metadata.Error == nil {
t.Error("Expected pull to fail")
}
}
func TestRetriesOnError(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockGetter := mock_ec2.NewMockHttpClient(ctrl)
testClient := ec2.NewEC2MetadataClient(mockGetter)
testTime := ttime.NewTestTime()
testTime.LudicrousSpeed(true)
ttime.SetTime(testTime)
gomock.InOrder(
mockGetter.EXPECT().Get(ec2.EC2_METADATA_SERVICE_URL+ec2.INSTANCE_IDENTITY_DOCUMENT_RESOURCE).Return(nil, errors.New("Something broke")),
mockGetter.EXPECT().Get(ec2.EC2_METADATA_SERVICE_URL+ec2.INSTANCE_IDENTITY_DOCUMENT_RESOURCE).Return(testErrorResponse()),
mockGetter.EXPECT().Get(ec2.EC2_METADATA_SERVICE_URL+ec2.INSTANCE_IDENTITY_DOCUMENT_RESOURCE).Return(testSuccessResponse(testInstanceIdentityDoc)),
)
doc, err := testClient.InstanceIdentityDocument()
if err != nil {
t.Fatal("Expected to be able to get doc")
}
if doc.Region != "us-east-1" {
t.Error("Wrong region; expected us-east-1 but got " + doc.Region)
}
}
"github.com/aws/amazon-ecs-agent/agent/api"
"github.com/aws/amazon-ecs-agent/agent/config"
"github.com/aws/amazon-ecs-agent/agent/utils"
"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
docker "github.com/fsouza/go-dockerclient"
)
var testRegistryHost = "127.0.0.1:51670"
var testRegistryImage = "127.0.0.1:51670/amazon/amazon-ecs-netkitten:latest"
var testAuthRegistryHost = "127.0.0.1:51671"
var testAuthRegistryImage = "127.0.0.1:51671/amazon/amazon-ecs-netkitten:latest"
var testVolumeImage = "127.0.0.1:51670/amazon/amazon-ecs-volumes-test:latest"
var testAuthUser = "******"
var testAuthPass = "******"
var test_time = ttime.NewTestTime()
var taskEngine *DockerTaskEngine
var initOnce sync.Once
func setup(t *testing.T) TaskEngine {
if testing.Short() {
t.Skip("Skipping integ test in short mode")
}
if _, err := os.Stat("/var/run/docker.sock"); err != nil {
t.Skip("Docker not running")
}
if os.Getenv("ECS_SKIP_ENGINE_INTEG_TEST") != "" {
t.Skip("ECS_SKIP_ENGINE_INTEG_TEST")
}
"errors"
"reflect"
"sync"
"testing"
"time"
"github.com/aws/amazon-ecs-agent/agent/api"
"github.com/aws/amazon-ecs-agent/agent/config"
"github.com/aws/amazon-ecs-agent/agent/engine/dockerclient"
"github.com/aws/amazon-ecs-agent/agent/engine/testdata"
"github.com/aws/amazon-ecs-agent/agent/statemanager/mocks"
"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
"github.com/golang/mock/gomock"
)
var dte_test_time = ttime.NewTestTime()
var defaultConfig = config.DefaultConfig()
func mocks(t *testing.T, cfg *config.Config) (*gomock.Controller, *MockDockerClient, TaskEngine) {
ctrl := gomock.NewController(t)
client := NewMockDockerClient(ctrl)
taskEngine := NewTaskEngine(cfg, false)
taskEngine.(*DockerTaskEngine).SetDockerClient(client)
return ctrl, client, taskEngine
}
func TestBatchContainerHappyPath(t *testing.T) {
ctrl, client, taskEngine := mocks(t, &defaultConfig)
defer ctrl.Finish()
ttime.SetTime(dte_test_time)
func TestHandlerDoesntLeakGouroutines(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
taskEngine := engine.NewMockTaskEngine(ctrl)
ecsclient := mock_api.NewMockECSClient(ctrl)
statemanager := statemanager.NewNoopStateManager()
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
closeWS := make(chan bool)
server, serverIn, requests, errs, err := startMockAcsServer(t, closeWS)
if err != nil {
t.Fatal(err)
}
go func() {
for {
select {
case <-requests:
case <-errs:
}
}
}()
timesConnected := 0
ecsclient.EXPECT().DiscoverPollEndpoint("myArn").Return(server.URL, nil).AnyTimes().Do(func(_ interface{}) {
timesConnected++
})
taskEngine.EXPECT().Version().Return("Docker: 1.5.0", nil).AnyTimes()
taskEngine.EXPECT().AddTask(gomock.Any()).AnyTimes()
ctx, cancel := context.WithCancel(context.Background())
ended := make(chan bool, 1)
go func() {
handler.StartSession(ctx, handler.StartSessionArguments{"myArn", credentials.AnonymousCredentials, &config.Config{Cluster: "someCluster"}, taskEngine, ecsclient, statemanager, true})
ended <- true
}()
// Warm it up
serverIn <- `{"type":"HeartbeatMessage","message":{"healthy":true}}`
serverIn <- samplePayloadMessage
beforeGoroutines := runtime.NumGoroutine()
for i := 0; i < 100; i++ {
serverIn <- `{"type":"HeartbeatMessage","message":{"healthy":true}}`
serverIn <- samplePayloadMessage
closeWS <- true
}
cancel()
testTime.Cancel()
<-ended
afterGoroutines := runtime.NumGoroutine()
t.Logf("Gorutines after 1 and after 100 acs messages: %v and %v", beforeGoroutines, afterGoroutines)
if timesConnected < 50 {
t.Fatal("Expected times connected to be a large number, was ", timesConnected)
}
if afterGoroutines > beforeGoroutines+5 {
t.Error("Goroutine leak, oh no!")
pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
}
}
func TestHeartbeatOnlyWhenIdle(t *testing.T) {
testTime := ttime.NewTestTime()
ttime.SetTime(testTime)
ctrl := gomock.NewController(t)
defer ctrl.Finish()
taskEngine := engine.NewMockTaskEngine(ctrl)
ecsclient := mock_api.NewMockECSClient(ctrl)
statemanager := statemanager.NewNoopStateManager()
closeWS := make(chan bool)
server, serverIn, requestsChan, errChan, err := startMockAcsServer(t, closeWS)
defer close(serverIn)
go func() {
for {
<-requestsChan
}
}()
if err != nil {
t.Fatal(err)
}
// We're testing that it does not reconnect here; must be the case
ecsclient.EXPECT().DiscoverPollEndpoint("myArn").Return(server.URL, nil).Times(1)
taskEngine.EXPECT().Version().Return("Docker: 1.5.0", nil).AnyTimes()
ctx, cancel := context.WithCancel(context.Background())
ended := make(chan bool, 1)
go func() {
handler.StartSession(ctx, handler.StartSessionArguments{
ContainerInstanceArn: "myArn",
CredentialProvider: credentials.AnonymousCredentials,
Config: &config.Config{Cluster: "someCluster"},
TaskEngine: taskEngine,
ECSClient: ecsclient,
StateManager: statemanager,
AcceptInvalidCert: true,
})
ended <- true
}()
taskAdded := make(chan bool)
taskEngine.EXPECT().AddTask(gomock.Any()).Do(func(interface{}) {
taskAdded <- true
}).Times(10)
for i := 0; i < 10; i++ {
serverIn <- samplePayloadMessage
testTime.Warp(1 * time.Minute)
<-taskAdded
}
select {
case <-ended:
t.Fatal("Should not have stop session")
case err := <-errChan:
t.Fatal("Error should not have been returned from server", err)
default:
}
go server.Close()
cancel()
<-ended
}