// createStack creates a new CloudFormation stack with the given input. This
// function returns as soon as the stack creation has been submitted. It does
// not wait for the stack creation to complete.
func (s *Scheduler) createStack(ctx context.Context, input *createStackInput, output chan stackOperationOutput, ss scheduler.StatusStream) error {
waiter := func(input *cloudformation.DescribeStacksInput) error {
scheduler.Publish(ctx, ss, "Creating stack")
err := s.cloudformation.WaitUntilStackCreateComplete(input)
if err == nil {
scheduler.Publish(ctx, ss, "Stack created")
}
return err
}
submitted := make(chan error)
fn := func() error {
_, err := s.cloudformation.CreateStack(&cloudformation.CreateStackInput{
StackName: input.StackName,
TemplateURL: input.Template.URL,
Tags: input.Tags,
Parameters: input.Parameters,
})
submitted <- err
return err
}
go func() {
stack, err := s.performStackOperation(ctx, *input.StackName, fn, waiter, ss)
output <- stackOperationOutput{stack, err}
}()
return <-submitted
}
// waitFor returns a wait function that will wait for the given stack operation
// to complete, and sends status messages to the status stream, and also records
// metrics for how long the operation took.
func (s *Scheduler) waitFor(ctx context.Context, op stackOperation, ss scheduler.StatusStream) func(*cloudformation.DescribeStacksInput) error {
waiter := waiters[op]
wait := waiter.wait(s.cloudformation)
return func(input *cloudformation.DescribeStacksInput) error {
tags := []string{
fmt.Sprintf("stack:%s", *input.StackName),
}
scheduler.Publish(ctx, ss, waiter.startMessage)
start := time.Now()
err := wait(input)
stats.Timing(ctx, fmt.Sprintf("scheduler.cloudformation.%s", op), time.Since(start), 1.0, tags)
if err == nil {
scheduler.Publish(ctx, ss, waiter.successMessage)
}
return err
}
}
// updateStack updates an existing CloudFormation stack with the given input.
// If there are no other active updates, this function returns as soon as the
// stack update has been submitted. If there are other updates, the function
// returns after `lockTimeout` and the update continues in the background.
func (s *Scheduler) updateStack(ctx context.Context, input *updateStackInput, output chan stackOperationOutput, ss scheduler.StatusStream) error {
waiter := func(input *cloudformation.DescribeStacksInput) error {
scheduler.Publish(ctx, ss, "Waiting for stack update to complete")
err := s.cloudformation.WaitUntilStackUpdateComplete(input)
if err == nil {
scheduler.Publish(ctx, ss, "Stack update complete")
}
return err
}
locked := make(chan struct{})
submitted := make(chan error, 1)
fn := func() error {
close(locked)
err := s.executeStackUpdate(input)
if err == nil {
scheduler.Publish(ctx, ss, "Stack update submitted")
}
submitted <- err
return err
}
go func() {
stack, err := s.performStackOperation(ctx, *input.StackName, fn, waiter, ss)
output <- stackOperationOutput{stack, err}
}()
var err error
select {
case <-s.after(lockWait):
scheduler.Publish(ctx, ss, "Waiting for existing stack operation to complete")
// FIXME: At this point, we don't want to affect UX by waiting
// around, so we return. But, if the stack update times out, or
// there's an error, that information is essentially silenced.
return nil
case <-locked:
// if a lock is obtained within the time frame, we might as well
// just wait for the update to get submitted.
err = <-submitted
}
return err
}
func (s *Scheduler) waitUntilStable(ctx context.Context, stack *cloudformation.Stack, ss scheduler.StatusStream) error {
deployments, err := deploymentsToWatch(stack)
if err != nil {
return err
}
deploymentStatuses := s.waitForDeploymentsToStabilize(ctx, deployments)
for status := range deploymentStatuses {
scheduler.Publish(ctx, ss, fmt.Sprintf("Service %s became %s", status.deployment.process, status))
}
// TODO publish notification to empire
return nil
}
// performStackOperation encapsulates the process of obtaining the stack
// operation lock, performing the stack operation, waiting for it to complete,
// then unlocking the stack operation lock.
//
// * If there are no operations currently in progress, the stack operation will execute.
// * If there is a currently active stack operation, this operation will wait
// until the other stack operation has completed.
// * If there is another pending stack operation, it will be replaced by the new
// update.
func (s *Scheduler) performStackOperation(ctx context.Context, stackName string, fn func() error, waiter func(*cloudformation.DescribeStacksInput) error, ss scheduler.StatusStream) (*cloudformation.Stack, error) {
l, err := newAdvisoryLock(s.db, stackName)
if err != nil {
return nil, err
}
// Cancel any pending stack operation, since this one obsoletes older
// operations.
if err := l.CancelPending(); err != nil {
return nil, fmt.Errorf("error canceling pending stack operation: %v", err)
}
if err := l.Lock(); err != nil {
// This will happen when a newer stack update obsoletes
// this one. We simply return nil.
//
// TODO: Should we return an error here?
if err == pglock.Canceled {
scheduler.Publish(ctx, ss, "Operation superseded by newer release")
return nil, nil
}
return nil, fmt.Errorf("error obtaining stack operation lock %s: %v", stackName, err)
}
defer l.Unlock()
// Once the lock has been obtained, let's perform the stack operation.
if err := fn(); err != nil {
return nil, err
}
wait := func() error {
return waiter(&cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
})
}
// Wait until this stack operation has completed. The lock will be
// unlocked when this function returns.
if err := s.waitUntilStackOperationComplete(l, wait); err != nil {
return nil, err
}
return s.stack(&stackName)
}
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
stackName, err := s.stackName(app.ID)
if err == errNoStack {
t := s.StackNameTemplate
if t == nil {
t = DefaultStackNameTemplate
}
buf := new(bytes.Buffer)
if err := t.Execute(buf, app); err != nil {
return fmt.Errorf("error generating stack name: %v", err)
}
stackName = buf.String()
if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
return err
}
} else if err != nil {
return err
}
t, err := s.createTemplate(ctx, app)
if err != nil {
return err
}
scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))
tags := append(s.Tags,
&cloudformation.Tag{Key: aws.String("empire.app.id"), Value: aws.String(app.ID)},
&cloudformation.Tag{Key: aws.String("empire.app.name"), Value: aws.String(app.Name)},
)
// Build parameters for the stack.
parameters := []*cloudformation.Parameter{
// FIXME: Remove this in favor of a Restart method.
{
ParameterKey: aws.String(restartParameter),
ParameterValue: aws.String(newUUID()),
},
}
if opts.NoDNS != nil {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String("DNS"),
ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
})
}
for _, p := range app.Processes {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String(scaleParameter(p.Type)),
ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
})
}
output := make(chan stackOperationOutput, 1)
_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
})
if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
if err := s.createStack(ctx, &createStackInput{
StackName: aws.String(stackName),
Template: t,
Tags: tags,
Parameters: parameters,
}, output, ss); err != nil {
return fmt.Errorf("error creating stack: %v", err)
}
} else if err == nil {
if err := s.updateStack(ctx, &updateStackInput{
StackName: aws.String(stackName),
Template: t,
Parameters: parameters,
// TODO: Update Go client
// Tags: tags,
}, output, ss); err != nil {
return err
}
} else {
return fmt.Errorf("error describing stack: %v", err)
}
if ss != nil {
o := <-output
if o.err != nil || o.stack == nil {
return o.err
}
if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
}
}
return nil
}
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
stackName, err := s.stackName(app.ID)
if err == errNoStack {
t := s.StackNameTemplate
if t == nil {
t = DefaultStackNameTemplate
}
buf := new(bytes.Buffer)
if err := t.Execute(buf, app); err != nil {
return fmt.Errorf("error generating stack name: %v", err)
}
stackName = buf.String()
if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
return err
}
} else if err != nil {
return err
}
stackTags := append(s.Tags, tagsFromLabels(app.Labels)...)
t, err := s.createTemplate(ctx, app, stackTags)
if err != nil {
return err
}
stats.Histogram(ctx, "scheduler.cloudformation.template_size", float32(t.Size), 1.0, []string{
fmt.Sprintf("stack:%s", stackName),
})
scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))
var parameters []*cloudformation.Parameter
if opts.NoDNS != nil {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String("DNS"),
ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
})
}
for _, p := range app.Processes {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String(scaleParameter(p.Type)),
ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
})
}
output := make(chan stackOperationOutput, 1)
_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
})
if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
if err := s.createStack(ctx, &createStackInput{
StackName: aws.String(stackName),
Template: t,
Tags: stackTags,
Parameters: parameters,
}, output, ss); err != nil {
return fmt.Errorf("error creating stack: %v", err)
}
} else if err == nil {
if err := s.updateStack(ctx, &updateStackInput{
StackName: aws.String(stackName),
Template: t,
Parameters: parameters,
Tags: stackTags,
}, output, ss); err != nil {
return err
}
} else {
return fmt.Errorf("error describing stack: %v", err)
}
if ss != nil {
o := <-output
if o.err != nil || o.stack == nil {
return o.err
}
if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
}
}
return nil
}
