Beispiel #1
0
func Publish(ctx context.Context, stream StatusStream, msg string) {
	if stream != nil {
		if err := stream.Publish(Status{Message: msg}); err != nil {
			logger.Warn(ctx, fmt.Sprintf("error publishing to stream: %v", err))
		}
	}
}
Beispiel #2
0
func (s *Scheduler) waitForDeploymentsToStabilize(ctx context.Context, deployments map[string]*ecsDeployment) <-chan *deploymentStatus {
	ch := make(chan *deploymentStatus)

	wait := func(deployments map[string]*ecsDeployment) (bool, error) {
		arns := make([]*string, 0, len(deployments))
		for arn := range deployments {
			arns = append(arns, aws.String(arn))
		}

		status, err := s.ecs.DescribeServices(&ecs.DescribeServicesInput{
			Cluster:  aws.String(s.Cluster),
			Services: arns,
		})
		if err != nil {
			return false, err
		}

		for _, service := range status.Services {
			d, ok := deployments[*service.ServiceArn]
			if !ok {
				return false, fmt.Errorf("missing deployment for: %s", service.ServiceArn)
			}
			primary := false
			stable := len(service.Deployments) == 1
			for _, deployment := range service.Deployments {
				if *deployment.Id == d.ID {
					primary = *deployment.Status == "PRIMARY"
				}
			}

			if primary && stable {
				ch <- &deploymentStatus{d, "stable"}
				delete(deployments, *service.ServiceArn)
			} else if primary {
				// do nothing
			} else {
				ch <- &deploymentStatus{d, "inactive"}
				return false, nil
			}
		}
		return true, nil
	}

	go func(deployments map[string]*ecsDeployment) {
		keepWaiting := true
		var err error
		for keepWaiting && len(deployments) > 0 {
			keepWaiting, err = wait(deployments)
			if err != nil {
				logger.Warn(ctx, fmt.Sprintf("error waiting for services to stabilize: %v", err))
				break
			}
		}
		close(ch)
	}(deployments)

	return ch
}
Beispiel #3
0
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
	stackName, err := s.stackName(app.ID)
	if err == errNoStack {
		t := s.StackNameTemplate
		if t == nil {
			t = DefaultStackNameTemplate
		}
		buf := new(bytes.Buffer)
		if err := t.Execute(buf, app); err != nil {
			return fmt.Errorf("error generating stack name: %v", err)
		}
		stackName = buf.String()
		if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}

	t, err := s.createTemplate(ctx, app)
	if err != nil {
		return err
	}

	scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))

	tags := append(s.Tags,
		&cloudformation.Tag{Key: aws.String("empire.app.id"), Value: aws.String(app.ID)},
		&cloudformation.Tag{Key: aws.String("empire.app.name"), Value: aws.String(app.Name)},
	)

	// Build parameters for the stack.
	parameters := []*cloudformation.Parameter{
		// FIXME: Remove this in favor of a Restart method.
		{
			ParameterKey:   aws.String(restartParameter),
			ParameterValue: aws.String(newUUID()),
		},
	}

	if opts.NoDNS != nil {
		parameters = append(parameters, &cloudformation.Parameter{
			ParameterKey:   aws.String("DNS"),
			ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
		})
	}

	for _, p := range app.Processes {
		parameters = append(parameters, &cloudformation.Parameter{
			ParameterKey:   aws.String(scaleParameter(p.Type)),
			ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
		})
	}

	output := make(chan stackOperationOutput, 1)
	_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
		StackName: aws.String(stackName),
	})
	if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
		if err := s.createStack(ctx, &createStackInput{
			StackName:  aws.String(stackName),
			Template:   t,
			Tags:       tags,
			Parameters: parameters,
		}, output, ss); err != nil {
			return fmt.Errorf("error creating stack: %v", err)
		}
	} else if err == nil {
		if err := s.updateStack(ctx, &updateStackInput{
			StackName:  aws.String(stackName),
			Template:   t,
			Parameters: parameters,
			// TODO: Update Go client
			// Tags:         tags,
		}, output, ss); err != nil {
			return err
		}
	} else {
		return fmt.Errorf("error describing stack: %v", err)
	}

	if ss != nil {
		o := <-output
		if o.err != nil || o.stack == nil {
			return o.err
		}
		if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
			logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
		}
	}
	return nil
}
Beispiel #4
0
// Submit creates (or updates) the CloudFormation stack for the app.
func (s *Scheduler) submit(ctx context.Context, tx *sql.Tx, app *scheduler.App, ss scheduler.StatusStream, opts SubmitOptions) error {
	stackName, err := s.stackName(app.ID)
	if err == errNoStack {
		t := s.StackNameTemplate
		if t == nil {
			t = DefaultStackNameTemplate
		}
		buf := new(bytes.Buffer)
		if err := t.Execute(buf, app); err != nil {
			return fmt.Errorf("error generating stack name: %v", err)
		}
		stackName = buf.String()
		if _, err := tx.Exec(`INSERT INTO stacks (app_id, stack_name) VALUES ($1, $2)`, app.ID, stackName); err != nil {
			return err
		}
	} else if err != nil {
		return err
	}

	stackTags := append(s.Tags, tagsFromLabels(app.Labels)...)

	t, err := s.createTemplate(ctx, app, stackTags)
	if err != nil {
		return err
	}

	stats.Histogram(ctx, "scheduler.cloudformation.template_size", float32(t.Size), 1.0, []string{
		fmt.Sprintf("stack:%s", stackName),
	})

	scheduler.Publish(ctx, ss, fmt.Sprintf("Created cloudformation template: %v (%d/%d bytes)", *t.URL, t.Size, MaxTemplateSize))

	var parameters []*cloudformation.Parameter
	if opts.NoDNS != nil {
		parameters = append(parameters, &cloudformation.Parameter{
			ParameterKey:   aws.String("DNS"),
			ParameterValue: aws.String(fmt.Sprintf("%t", *opts.NoDNS)),
		})
	}

	for _, p := range app.Processes {
		parameters = append(parameters, &cloudformation.Parameter{
			ParameterKey:   aws.String(scaleParameter(p.Type)),
			ParameterValue: aws.String(fmt.Sprintf("%d", p.Instances)),
		})
	}

	output := make(chan stackOperationOutput, 1)
	_, err = s.cloudformation.DescribeStacks(&cloudformation.DescribeStacksInput{
		StackName: aws.String(stackName),
	})
	if err, ok := err.(awserr.Error); ok && err.Message() == fmt.Sprintf("Stack with id %s does not exist", stackName) {
		if err := s.createStack(ctx, &createStackInput{
			StackName:  aws.String(stackName),
			Template:   t,
			Tags:       stackTags,
			Parameters: parameters,
		}, output, ss); err != nil {
			return fmt.Errorf("error creating stack: %v", err)
		}
	} else if err == nil {
		if err := s.updateStack(ctx, &updateStackInput{
			StackName:  aws.String(stackName),
			Template:   t,
			Parameters: parameters,
			Tags:       stackTags,
		}, output, ss); err != nil {
			return err
		}
	} else {
		return fmt.Errorf("error describing stack: %v", err)
	}

	if ss != nil {
		o := <-output
		if o.err != nil || o.stack == nil {
			return o.err
		}
		if err := s.waitUntilStable(ctx, o.stack, ss); err != nil {
			logger.Warn(ctx, fmt.Sprintf("error waiting for submit to stabilize: %v", err))
		}
	}
	return nil
}