func (t *Test) invokeLambdas(awsConfig *aws.Config, sqsURL string) {
lambdas := numberOfLambdas(t.config.Concurrency, len(t.config.Regions))
for i := 0; i < lambdas; i++ {
region := t.config.Regions[i%len(t.config.Regions)]
requests, requestsRemainder := divide(t.config.TotalRequests, lambdas)
concurrency, _ := divide(t.config.Concurrency, lambdas)
if requestsRemainder > 0 && i == lambdas-1 {
requests += requestsRemainder
}
c := t.config
args := invokeArgs{
File: "./goad-lambda",
Args: []string{
c.URL,
strconv.Itoa(int(concurrency)),
strconv.Itoa(int(requests)),
sqsURL,
region,
c.RequestTimeout.String(),
reportingFrequency(lambdas).String(),
c.Regions[0],
c.Method,
},
}
config := aws.NewConfig().WithRegion(region)
go t.invokeLambda(config, args)
}
}
func TestClientOverrideDefaultHTTPClientTimeoutRace(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("us-east-1a"))
}))
cfg := aws.NewConfig().WithEndpoint(server.URL)
runEC2MetadataClients(t, cfg, 100)
}
// Config returns the default configuration without credentials.
// To retrieve a config with credentials also included use
// `defaults.Get().Config` instead.
//
// Generally you shouldn't need to use this method directly, but
// is available if you need to reset the configuration of an
// existing service client or session.
func Config() *aws.Config {
return aws.NewConfig().
WithCredentials(credentials.AnonymousCredentials).
WithRegion(os.Getenv("AWS_REGION")).
WithHTTPClient(http.DefaultClient).
WithMaxRetries(aws.UseServiceDefaultRetries).
WithLogger(aws.NewDefaultLogger()).
WithLogLevel(aws.LogOff).
WithSleepDelay(time.Sleep)
}
func TestValidateEndpointHandler(t *testing.T) {
os.Clearenv()
svc := awstesting.NewClient(aws.NewConfig().WithRegion("us-west-2"))
svc.Handlers.Clear()
svc.Handlers.Validate.PushBackNamed(corehandlers.ValidateEndpointHandler)
req := svc.NewRequest(&request.Operation{Name: "Operation"}, nil, nil)
err := req.Build()
assert.NoError(t, err)
}
func TestRequestExhaustRetries(t *testing.T) {
delays := []time.Duration{}
sleepDelay := func(delay time.Duration) {
delays = append(delays, delay)
}
reqNum := 0
reqs := []http.Response{
{StatusCode: 500, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
{StatusCode: 500, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
{StatusCode: 500, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
{StatusCode: 500, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
}
s := awstesting.NewClient(aws.NewConfig().WithSleepDelay(sleepDelay))
s.Handlers.Validate.Clear()
s.Handlers.Unmarshal.PushBack(unmarshal)
s.Handlers.UnmarshalError.PushBack(unmarshalError)
s.Handlers.Send.Clear() // mock sending
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &reqs[reqNum]
reqNum++
})
r := s.NewRequest(&request.Operation{Name: "Operation"}, nil, nil)
err := r.Send()
assert.NotNil(t, err)
if e, ok := err.(awserr.RequestFailure); ok {
assert.Equal(t, 500, e.StatusCode())
} else {
assert.Fail(t, "Expected error to be a service failure")
}
assert.Equal(t, "UnknownError", err.(awserr.Error).Code())
assert.Equal(t, "An error occurred.", err.(awserr.Error).Message())
assert.Equal(t, 3, int(r.RetryCount))
expectDelays := []struct{ min, max time.Duration }{{30, 59}, {60, 118}, {120, 236}}
for i, v := range delays {
min := expectDelays[i].min * time.Millisecond
max := expectDelays[i].max * time.Millisecond
assert.True(t, min <= v && v <= max,
"Expect delay to be within range, i:%d, v:%s, min:%s, max:%s", i, v, min, max)
}
}
func (infra *Infrastructure) createLambdaFunction(region, roleArn string, payload []byte) error {
config := aws.NewConfig().WithRegion(region)
svc := lambda.New(session.New(), config)
_, err := svc.GetFunction(&lambda.GetFunctionInput{
FunctionName: aws.String("goad"),
})
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ResourceNotFoundException" {
_, err := svc.CreateFunction(&lambda.CreateFunctionInput{
Code: &lambda.FunctionCode{
ZipFile: payload,
},
FunctionName: aws.String("goad"),
Handler: aws.String("index.handler"),
Role: aws.String(roleArn),
Runtime: aws.String("nodejs"),
MemorySize: aws.Int64(1536),
Publish: aws.Bool(true),
Timeout: aws.Int64(300),
})
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
// Calling this function too soon after creating the role might
// fail, so we should retry after a little while.
// TODO: limit the number of retries.
if awsErr.Code() == "InvalidParameterValueException" {
time.Sleep(time.Second)
return infra.createLambdaFunction(region, roleArn, payload)
}
}
return err
}
}
}
}
return nil
}
// Start a test
func (t *Test) Start() <-chan queue.RegionsAggData {
awsConfig := aws.NewConfig().WithRegion(t.config.Regions[0])
infra, err := infrastructure.New(t.config.Regions, awsConfig)
if err != nil {
log.Fatal(err)
}
t.invokeLambdas(awsConfig, infra.QueueURL())
results := make(chan queue.RegionsAggData)
go func() {
for result := range queue.Aggregate(awsConfig, infra.QueueURL(), t.config.TotalRequests) {
results <- result
}
infra.Clean()
close(results)
}()
return results
}
// test that retries don't occur for 4xx status codes with a response type that can't be retried
func TestRequest4xxUnretryable(t *testing.T) {
s := awstesting.NewClient(aws.NewConfig().WithMaxRetries(10))
s.Handlers.Validate.Clear()
s.Handlers.Unmarshal.PushBack(unmarshal)
s.Handlers.UnmarshalError.PushBack(unmarshalError)
s.Handlers.Send.Clear() // mock sending
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &http.Response{StatusCode: 401, Body: body(`{"__type":"SignatureDoesNotMatch","message":"Signature does not match."}`)}
})
out := &testData{}
r := s.NewRequest(&request.Operation{Name: "Operation"}, nil, out)
err := r.Send()
assert.NotNil(t, err)
if e, ok := err.(awserr.RequestFailure); ok {
assert.Equal(t, 401, e.StatusCode())
} else {
assert.Fail(t, "Expected error to be a service failure")
}
assert.Equal(t, "SignatureDoesNotMatch", err.(awserr.Error).Code())
assert.Equal(t, "Signature does not match.", err.(awserr.Error).Message())
assert.Equal(t, 0, int(r.RetryCount))
}
// test that retries occur for 4xx status codes with a response type that can be retried - see `shouldRetry`
func TestRequestRecoverRetry4xxRetryable(t *testing.T) {
reqNum := 0
reqs := []http.Response{
{StatusCode: 400, Body: body(`{"__type":"Throttling","message":"Rate exceeded."}`)},
{StatusCode: 429, Body: body(`{"__type":"ProvisionedThroughputExceededException","message":"Rate exceeded."}`)},
{StatusCode: 200, Body: body(`{"data":"valid"}`)},
}
s := awstesting.NewClient(aws.NewConfig().WithMaxRetries(10))
s.Handlers.Validate.Clear()
s.Handlers.Unmarshal.PushBack(unmarshal)
s.Handlers.UnmarshalError.PushBack(unmarshalError)
s.Handlers.Send.Clear() // mock sending
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &reqs[reqNum]
reqNum++
})
out := &testData{}
r := s.NewRequest(&request.Operation{Name: "Operation"}, nil, out)
err := r.Send()
assert.Nil(t, err)
assert.Equal(t, 2, int(r.RetryCount))
assert.Equal(t, "valid", out.Data)
}
// test that retries occur for 5xx status codes
func TestRequestRecoverRetry5xx(t *testing.T) {
reqNum := 0
reqs := []http.Response{
{StatusCode: 500, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
{StatusCode: 501, Body: body(`{"__type":"UnknownError","message":"An error occurred."}`)},
{StatusCode: 200, Body: body(`{"data":"valid"}`)},
}
s := awstesting.NewClient(aws.NewConfig().WithMaxRetries(10))
s.Handlers.Validate.Clear()
s.Handlers.Unmarshal.PushBack(unmarshal)
s.Handlers.UnmarshalError.PushBack(unmarshalError)
s.Handlers.Send.Clear() // mock sending
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &reqs[reqNum]
reqNum++
})
out := &testData{}
r := s.NewRequest(&request.Operation{Name: "Operation"}, nil, out)
err := r.Send()
assert.Nil(t, err)
assert.Equal(t, 2, int(r.RetryCount))
assert.Equal(t, "valid", out.Data)
}
func runLoadTest(client *http.Client, sqsurl string, url string, totalRequests int, concurrencycount int, awsregion string, reportingFrequency time.Duration, queueRegion string, requestMethod string) {
awsConfig := aws.NewConfig().WithRegion(queueRegion)
sqsAdaptor := queue.NewSQSAdaptor(awsConfig, sqsurl)
//sqsAdaptor := queue.NewDummyAdaptor(sqsurl)
jobs := make(chan struct{}, totalRequests)
ch := make(chan RequestResult, totalRequests)
var wg sync.WaitGroup
loadTestStartTime := time.Now()
var requestsSoFar int
for i := 0; i < totalRequests; i++ {
jobs <- struct{}{}
}
close(jobs)
fmt.Print("Spawning workers…")
for i := 0; i < concurrencycount; i++ {
wg.Add(1)
go fetch(loadTestStartTime, client, url, totalRequests, jobs, ch, &wg, awsregion, requestMethod)
fmt.Print(".")
}
fmt.Println(" done.\nWaiting for results…")
ticker := time.NewTicker(reportingFrequency)
quit := make(chan struct{})
quitting := false
for requestsSoFar < totalRequests && !quitting {
i := 0
var timeToFirstTotal int64
var requestTimeTotal int64
totBytesRead := 0
statuses := make(map[string]int)
var firstRequestTime int64
var lastRequestTime int64
var slowest int64
var fastest int64
var totalTimedOut int
var totalConnectionError int
resetStats := false
for requestsSoFar < totalRequests && !quitting && !resetStats {
aggregate := false
select {
case r := <-ch:
i++
requestsSoFar++
if requestsSoFar%10 == 0 || requestsSoFar == totalRequests {
fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(requestsSoFar)/float64(totalRequests))*100.0, requestsSoFar, totalRequests)
}
if firstRequestTime == 0 {
firstRequestTime = r.Time
}
lastRequestTime = r.Time
if r.Timeout {
totalTimedOut++
continue
}
if r.ConnectionError {
totalConnectionError++
continue
}
if r.ElapsedLastByte > slowest {
slowest = r.ElapsedLastByte
}
if fastest == 0 {
fastest = r.ElapsedLastByte
} else {
if r.ElapsedLastByte < fastest {
fastest = r.ElapsedLastByte
}
}
timeToFirstTotal += r.ElapsedFirstByte
totBytesRead += r.Bytes
statusStr := strconv.Itoa(r.Status)
_, ok := statuses[statusStr]
if !ok {
statuses[statusStr] = 1
} else {
statuses[statusStr]++
}
requestTimeTotal += r.Elapsed
if requestsSoFar == totalRequests {
quitting = true
}
case <-ticker.C:
if i == 0 {
continue
}
aggregate = true
case <-quit:
ticker.Stop()
quitting = true
}
if aggregate || quitting {
durationNanoSeconds := lastRequestTime - firstRequestTime
durationSeconds := float32(durationNanoSeconds) / float32(1000000000)
var reqPerSec float32
var kbPerSec float32
if durationSeconds > 0 {
reqPerSec = float32(i) / durationSeconds
kbPerSec = (float32(totBytesRead) / durationSeconds) / 1024.0
} else {
reqPerSec = 0
kbPerSec = 0
}
fatalError := ""
if (totalTimedOut + totalConnectionError) > i/2 {
fatalError = "Over 50% of requests failed, aborting"
quitting = true
}
aggData := queue.AggData{
i,
totalTimedOut,
totalConnectionError,
timeToFirstTotal / int64(i),
totBytesRead,
statuses,
requestTimeTotal / int64(i),
reqPerSec,
kbPerSec,
slowest,
fastest,
awsregion,
fatalError,
}
sqsAdaptor.SendResult(aggData)
resetStats = true
}
}
}
fmt.Printf("\nYay🎈 - %d requests completed\n", requestsSoFar)
}
func TestClientDisableOverrideDefaultHTTPClientTimeout(t *testing.T) {
svc := ec2metadata.New(session.New(aws.NewConfig().WithEC2MetadataDisableTimeoutOverride(true)))
assert.Equal(t, http.DefaultClient, svc.Config.HTTPClient)
}