// ECSRetryHandler defines how to retry ECS service calls. It behaves like the default retry handler, except for the SubmitStateChange operations where it has a massive upper limit on retry counts
func ECSRetryHandler(r *aws.Request) {
if r.Operation == nil || (r.Operation.Name != opSubmitContainerStateChange && r.Operation.Name != opSubmitTaskStateChange) {
aws.AfterRetryHandler(r)
return
}
// else this is a Submit*StateChange operation
// For these operations, fake the retry count for the sake of the WillRetry check.
// Do this by temporarily setting it to 0 before calling that check.
// We still keep the value around for sleep calculations
// See https://github.com/aws/aws-sdk-go/blob/b2d953f489cf94029392157225e893d7b69cd447/aws/handler_functions.go#L107
// for this code's inspiration
realRetryCount := r.RetryCount
if r.RetryCount < maxSubmitRetryCount {
r.RetryCount = 0
}
r.Retryable.Set(r.Service.ShouldRetry(r))
if r.WillRetry() {
r.RetryCount = realRetryCount
if r.RetryCount > 20 {
// Hardcoded max for calling RetryRules here because it *will* overflow if you let it and result in sleeping negative time
r.RetryDelay = maxSubmitRetryDelay
} else {
r.RetryDelay = durationMin(maxSubmitRetryDelay, r.Service.RetryRules(r))
}
// AddJitter is purely additive, so subtracting half the amount of jitter
// makes it average out to RetryDelay
ttime.Sleep(utils.AddJitter(r.RetryDelay-submitRetryDelayJitter/2, submitRetryDelayJitter))
if r.Error != nil {
if err, ok := r.Error.(awserr.Error); ok {
if isCodeExpiredCreds(err.Code()) {
r.Config.Credentials.Expire()
}
}
}
r.RetryCount++
r.Error = nil
}
}
// newSubmitStateChangeClient returns a client intended to be used for
// Submit*StateChange APIs which has the behavior of retrying the call on
// retriable errors for an extended period of time (roughly 24 hours).
func newSubmitStateChangeClient(awsConfig *aws.Config) *ecs.ECS {
sscConfig := awsConfig.Copy()
sscConfig.MaxRetries = submitStateChangeMaxDelayRetries
client := ecs.New(&sscConfig)
client.Handlers.AfterRetry.Clear()
client.Handlers.AfterRetry.PushBack(
extendedRetryMaxDelayHandlerFactory(submitStateChangeExtraRetries))
client.DefaultMaxRetries = submitStateChangeMaxDelayRetries
return client
}
var awsAfterRetryHandler = func(r *aws.Request) {
aws.AfterRetryHandler(r)
}
// ExtendedRetryMaxDelayHandlerFactory returns a function which can be used as an AfterRetryHandler
// in the AWS Go SDK. This AfterRetryHandler can be used to have a large number of retries where
// the initial delay between backoff grows exponentially (2^n * 30ms; defined in service.retryRules)
// and extra retries are performed at the maximum delay of the initial exponential growth.
func extendedRetryMaxDelayHandlerFactory(maxExtendedRetries uint) func(r *aws.Request) {
return func(r *aws.Request) {
realRetryCount := r.RetryCount
maxDelayRetries := r.MaxRetries()
if r.RetryCount >= (maxDelayRetries + maxExtendedRetries) {
return
} else if r.RetryCount >= maxDelayRetries {
r.RetryCount = maxDelayRetries - 1
}