func Publish(ctx context.Context, stream StatusStream, msg string) {
if stream != nil {
if err := stream.Publish(Status{Message: msg}); err != nil {
logger.Warn(ctx, fmt.Sprintf("error publishing to stream: %v", err))
}
}
}
func (s *Scheduler) waitForDeploymentsToStabilize(ctx context.Context, deployments map[string]*ecsDeployment) <-chan *deploymentStatus {
ch := make(chan *deploymentStatus)
wait := func(deployments map[string]*ecsDeployment) (bool, error) {
arns := make([]*string, 0, len(deployments))
for arn := range deployments {
arns = append(arns, aws.String(arn))
}
status, err := s.ecs.DescribeServices(&ecs.DescribeServicesInput{
Cluster: aws.String(s.Cluster),
Services: arns,
})
if err != nil {
return false, err
}
for _, service := range status.Services {
d, ok := deployments[*service.ServiceArn]
if !ok {
return false, fmt.Errorf("missing deployment for: %s", service.ServiceArn)
}
primary := false
stable := len(service.Deployments) == 1
for _, deployment := range service.Deployments {
if *deployment.Id == d.ID {
primary = *deployment.Status == "PRIMARY"
}
}
if primary && stable {
ch <- &deploymentStatus{d, "stable"}
delete(deployments, *service.ServiceArn)
} else if primary {
// do nothing
} else {
ch <- &deploymentStatus{d, "inactive"}
return false, nil
}
}
return true, nil
}
go func(deployments map[string]*ecsDeployment) {
keepWaiting := true
var err error
for keepWaiting && len(deployments) > 0 {
keepWaiting, err = wait(deployments)
if err != nil {
logger.Warn(ctx, fmt.Sprintf("error waiting for services to stabilize: %v", err))
break
}
}
close(ch)
}(deployments)
return ch
}
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
stackName, err := s.stackName(app.ID)
if err == errNoStack {
t := s.StackNameTemplate
if t == nil {
t = DefaultStackNameTemplate
}
buf := new(bytes.Buffer)
if err := t.Execute(buf, app); err != nil {
return fmt.Errorf("error generating stack name: %v", err)
}
stackName = buf.String()
if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
return err
}
} else if err != nil {
return err
}
t, err := s.createTemplate(ctx, app)
if err != nil {
return err
}
scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))
tags := append(s.Tags,
&cloudformation.Tag{Key: aws.String("empire.app.id"), Value: aws.String(app.ID)},
&cloudformation.Tag{Key: aws.String("empire.app.name"), Value: aws.String(app.Name)},
)
// Build parameters for the stack.
parameters := []*cloudformation.Parameter{
// FIXME: Remove this in favor of a Restart method.
{
ParameterKey: aws.String(restartParameter),
ParameterValue: aws.String(newUUID()),
},
}
if opts.NoDNS != nil {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String("DNS"),
ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
})
}
for _, p := range app.Processes {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String(scaleParameter(p.Type)),
ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
})
}
output := make(chan stackOperationOutput, 1)
_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
})
if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
if err := s.createStack(ctx, &createStackInput{
StackName: aws.String(stackName),
Template: t,
Tags: tags,
Parameters: parameters,
}, output, ss); err != nil {
return fmt.Errorf("error creating stack: %v", err)
}
} else if err == nil {
if err := s.updateStack(ctx, &updateStackInput{
StackName: aws.String(stackName),
Template: t,
Parameters: parameters,
// TODO: Update Go client
// Tags: tags,
}, output, ss); err != nil {
return err
}
} else {
return fmt.Errorf("error describing stack: %v", err)
}
if ss != nil {
o := <-output
if o.err != nil || o.stack == nil {
return o.err
}
if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
}
}
return nil
}
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
stackName, err := s.stackName(app.ID)
if err == errNoStack {
t := s.StackNameTemplate
if t == nil {
t = DefaultStackNameTemplate
}
buf := new(bytes.Buffer)
if err := t.Execute(buf, app); err != nil {
return fmt.Errorf("error generating stack name: %v", err)
}
stackName = buf.String()
if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
return err
}
} else if err != nil {
return err
}
stackTags := append(s.Tags, tagsFromLabels(app.Labels)...)
t, err := s.createTemplate(ctx, app, stackTags)
if err != nil {
return err
}
stats.Histogram(ctx, "scheduler.cloudformation.template_size", float32(t.Size), 1.0, []string{
fmt.Sprintf("stack:%s", stackName),
})
scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))
var parameters []*cloudformation.Parameter
if opts.NoDNS != nil {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String("DNS"),
ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
})
}
for _, p := range app.Processes {
parameters = append(parameters, &cloudformation.Parameter{
ParameterKey: aws.String(scaleParameter(p.Type)),
ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
})
}
output := make(chan stackOperationOutput, 1)
_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stackName),
})
if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
if err := s.createStack(ctx, &createStackInput{
StackName: aws.String(stackName),
Template: t,
Tags: stackTags,
Parameters: parameters,
}, output, ss); err != nil {
return fmt.Errorf("error creating stack: %v", err)
}
} else if err == nil {
if err := s.updateStack(ctx, &updateStackInput{
StackName: aws.String(stackName),
Template: t,
Parameters: parameters,
Tags: stackTags,
}, output, ss); err != nil {
return err
}
} else {
return fmt.Errorf("error describing stack: %v", err)
}
if ss != nil {
o := <-output
if o.err != nil || o.stack == nil {
return o.err
}
if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
}
}
return nil
}