// createStack creates a new CloudFormation stack with the given input. This // function returns as soon as the stack creation has been submitted. It does // not wait for the stack creation to complete. func (s *Scheduler) createStack(ctx context.Context, input *createStackInput, output chan stackOperationOutput, ss scheduler.StatusStream) error { waiter := func(input *cloudformation.DescribeStacksInput) error { scheduler.Publish(ctx, ss, "Creating stack") err := s.cloudformation.WaitUntilStackCreateComplete(input) if err == nil { scheduler.Publish(ctx, ss, "Stack created") } return err } submitted := make(chan error) fn := func() error { _, err := s.cloudformation.CreateStack(&cloudformation.CreateStackInput{ StackName: input.StackName, TemplateURL: input.Template.URL, Tags: input.Tags, Parameters: input.Parameters, }) submitted <- err return err } go func() { stack, err := s.performStackOperation(ctx, *input.StackName, fn, waiter, ss) output <- stackOperationOutput{stack, err} }() return <-submitted }
// waitFor returns a wait function that will wait for the given stack operation // to complete, and sends status messages to the status stream, and also records // metrics for how long the operation took. func (s *Scheduler) waitFor(ctx context.Context, op stackOperation, ss scheduler.StatusStream) func(*cloudformation.DescribeStacksInput) error { waiter := waiters[op] wait := waiter.wait(s.cloudformation) return func(input *cloudformation.DescribeStacksInput) error { tags := []string{ fmt.Sprintf("stack:%s", *input.StackName), } scheduler.Publish(ctx, ss, waiter.startMessage) start := time.Now() err := wait(input) stats.Timing(ctx, fmt.Sprintf("scheduler.cloudformation.%s", op), time.Since(start), 1.0, tags) if err == nil { scheduler.Publish(ctx, ss, waiter.successMessage) } return err } }
// updateStack updates an existing CloudFormation stack with the given input. // If there are no other active updates, this function returns as soon as the // stack update has been submitted. If there are other updates, the function // returns after `lockTimeout` and the update continues in the background. func (s *Scheduler) updateStack(ctx context.Context, input *updateStackInput, output chan stackOperationOutput, ss scheduler.StatusStream) error { waiter := func(input *cloudformation.DescribeStacksInput) error { scheduler.Publish(ctx, ss, "Waiting for stack update to complete") err := s.cloudformation.WaitUntilStackUpdateComplete(input) if err == nil { scheduler.Publish(ctx, ss, "Stack update complete") } return err } locked := make(chan struct{}) submitted := make(chan error, 1) fn := func() error { close(locked) err := s.executeStackUpdate(input) if err == nil { scheduler.Publish(ctx, ss, "Stack update submitted") } submitted <- err return err } go func() { stack, err := s.performStackOperation(ctx, *input.StackName, fn, waiter, ss) output <- stackOperationOutput{stack, err} }() var err error select { case <-s.after(lockWait): scheduler.Publish(ctx, ss, "Waiting for existing stack operation to complete") // FIXME: At this point, we don't want to affect UX by waiting // around, so we return. But, if the stack update times out, or // there's an error, that information is essentially silenced. return nil case <-locked: // if a lock is obtained within the time frame, we might as well // just wait for the update to get submitted. err = <-submitted } return err }
func (s *Scheduler) waitUntilStable(ctx context.Context, stack *cloudformation.Stack, ss scheduler.StatusStream) error { deployments, err := deploymentsToWatch(stack) if err != nil { return err } deploymentStatuses := s.waitForDeploymentsToStabilize(ctx, deployments) for status := range deploymentStatuses { scheduler.Publish(ctx, ss, fmt.Sprintf("Service %s became %s", status.deployment.process, status)) } // TODO publish notification to empire return nil }
// performStackOperation encapsulates the process of obtaining the stack // operation lock, performing the stack operation, waiting for it to complete, // then unlocking the stack operation lock. // // * If there are no operations currently in progress, the stack operation will execute. // * If there is a currently active stack operation, this operation will wait // until the other stack operation has completed. // * If there is another pending stack operation, it will be replaced by the new // update. func (s *Scheduler) performStackOperation(ctx context.Context, stackName string, fn func() error, waiter func(*cloudformation.DescribeStacksInput) error, ss scheduler.StatusStream) (*cloudformation.Stack, error) { l, err := newAdvisoryLock(s.db, stackName) if err != nil { return nil, err } // Cancel any pending stack operation, since this one obsoletes older // operations. if err := l.CancelPending(); err != nil { return nil, fmt.Errorf("error canceling pending stack operation: %v", err) } if err := l.Lock(); err != nil { // This will happen when a newer stack update obsoletes // this one. We simply return nil. // // TODO: Should we return an error here? if err == pglock.Canceled { scheduler.Publish(ctx, ss, "Operation superseded by newer release") return nil, nil } return nil, fmt.Errorf("error obtaining stack operation lock %s: %v", stackName, err) } defer l.Unlock() // Once the lock has been obtained, let's perform the stack operation. if err := fn(); err != nil { return nil, err } wait := func() error { return waiter(&cloudformation.DescribeStacksInput{ StackName: aws.String(stackName), }) } // Wait until this stack operation has completed. The lock will be // unlocked when this function returns. if err := s.waitUntilStackOperationComplete(l, wait); err != nil { return nil, err } return s.stack(&stackName) }
// Submit creates (or updates) the CloudFormation stack for the app. func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error { stackName, err := s.stackName(app.ID) if err == errNoStack { t := s.StackNameTemplate if t == nil { t = DefaultStackNameTemplate } buf := new(bytes.Buffer) if err := t.Execute(buf, app); err != nil { return fmt.Errorf("error generating stack name: %v", err) } stackName = buf.String() if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil { return err } } else if err != nil { return err } t, err := s.createTemplate(ctx, app) if err != nil { return err } scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize)) tags := append(s.Tags, &cloudformation.Tag{Key: aws.String("empire.app.id"), Value: aws.String(app.ID)}, &cloudformation.Tag{Key: aws.String("empire.app.name"), Value: aws.String(app.Name)}, ) // Build parameters for the stack. parameters := []*cloudformation.Parameter{ // FIXME: Remove this in favor of a Restart method. { ParameterKey: aws.String(restartParameter), ParameterValue: aws.String(newUUID()), }, } if opts.NoDNS != nil { parameters = append(parameters, &cloudformation.Parameter{ ParameterKey: aws.String("DNS"), ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)), }) } for _, p := range app.Processes { parameters = append(parameters, &cloudformation.Parameter{ ParameterKey: aws.String(scaleParameter(p.Type)), ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)), }) } output := make(chan stackOperationOutput, 1) _, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{ StackName: aws.String(stackName), }) if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) { if err := s.createStack(ctx, &createStackInput{ StackName: aws.String(stackName), Template: t, Tags: tags, Parameters: parameters, }, output, ss); err != nil { return fmt.Errorf("error creating stack: %v", err) } } else if err == nil { if err := s.updateStack(ctx, &updateStackInput{ StackName: aws.String(stackName), Template: t, Parameters: parameters, // TODO: Update Go client // Tags: tags, }, output, ss); err != nil { return err } } else { return fmt.Errorf("error describing stack: %v", err) } if ss != nil { o := <-output if o.err != nil || o.stack == nil { return o.err } if err := s.waitUntilStable(ctx, o.stack, ss); err != nil { logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err)) } } return nil }
// Submit creates (or updates) the CloudFormation stack for the app. func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error { stackName, err := s.stackName(app.ID) if err == errNoStack { t := s.StackNameTemplate if t == nil { t = DefaultStackNameTemplate } buf := new(bytes.Buffer) if err := t.Execute(buf, app); err != nil { return fmt.Errorf("error generating stack name: %v", err) } stackName = buf.String() if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil { return err } } else if err != nil { return err } stackTags := append(s.Tags, tagsFromLabels(app.Labels)...) t, err := s.createTemplate(ctx, app, stackTags) if err != nil { return err } stats.Histogram(ctx, "scheduler.cloudformation.template_size", float32(t.Size), 1.0, []string{ fmt.Sprintf("stack:%s", stackName), }) scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize)) var parameters []*cloudformation.Parameter if opts.NoDNS != nil { parameters = append(parameters, &cloudformation.Parameter{ ParameterKey: aws.String("DNS"), ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)), }) } for _, p := range app.Processes { parameters = append(parameters, &cloudformation.Parameter{ ParameterKey: aws.String(scaleParameter(p.Type)), ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)), }) } output := make(chan stackOperationOutput, 1) _, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{ StackName: aws.String(stackName), }) if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) { if err := s.createStack(ctx, &createStackInput{ StackName: aws.String(stackName), Template: t, Tags: stackTags, Parameters: parameters, }, output, ss); err != nil { return fmt.Errorf("error creating stack: %v", err) } } else if err == nil { if err := s.updateStack(ctx, &updateStackInput{ StackName: aws.String(stackName), Template: t, Parameters: parameters, Tags: stackTags, }, output, ss); err != nil { return err } } else { return fmt.Errorf("error describing stack: %v", err) } if ss != nil { o := <-output if o.err != nil || o.stack == nil { return o.err } if err := s.waitUntilStable(ctx, o.stack, ss); err != nil { logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err)) } } return nil }