func waitForCompletion(stack string, CloudFormation *cloudformation.CloudFormation, isDeleting bool) (string, error) {
for {
dres, err := CloudFormation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stack),
})
if err != nil {
stdcli.Error(err)
}
err = displayProgress(stack, CloudFormation, isDeleting)
if err != nil {
stdcli.Error(err)
}
if len(dres.Stacks) != 1 {
stdcli.Error(fmt.Errorf("could not read stack status"))
}
switch *dres.Stacks[0].StackStatus {
case "CREATE_COMPLETE":
// Dump .env if DEVELOPMENT
if isDevelopment {
fmt.Printf("Development .env:\n")
// convert Port5432TcpAddr to PORT_5432_TCP_ADDR
re := regexp.MustCompile("([a-z])([A-Z0-9])") // lower case letter followed by upper case or number, i.e. Port5432
re2 := regexp.MustCompile("([0-9])([A-Z])") // number followed by upper case letter, i.e. 5432Tcp
for _, o := range dres.Stacks[0].Outputs {
k := re.ReplaceAllString(*o.OutputKey, "${1}_${2}")
k = re2.ReplaceAllString(k, "${1}_${2}")
k = strings.ToUpper(k)
fmt.Printf("%v=%v\n", k, *o.OutputValue)
}
}
for _, o := range dres.Stacks[0].Outputs {
if *o.OutputKey == "Dashboard" {
return *o.OutputValue, nil
}
}
return "", fmt.Errorf("could not install stack, contact [email protected] for assistance")
case "CREATE_FAILED":
return "", fmt.Errorf("stack creation failed, contact [email protected] for assistance")
case "ROLLBACK_COMPLETE":
return "", fmt.Errorf("stack creation failed, contact [email protected] for assistance")
case "DELETE_COMPLETE":
return "", nil
case "DELETE_FAILED":
return "", fmt.Errorf("stack deletion failed, contact [email protected] for assistance")
}
time.Sleep(2 * time.Second)
}
}
// Does a given stack exist?
func stackExists(stackNameOrID string, cf *cloudformation.CloudFormation, logger *logrus.Logger) (bool, error) {
describeStacksInput := &cloudformation.DescribeStacksInput{
StackName: aws.String(stackNameOrID),
}
describeStacksOutput, err := cf.DescribeStacks(describeStacksInput)
logger.WithFields(logrus.Fields{
"DescribeStackOutput": describeStacksOutput,
}).Debug("DescribeStackOutput results")
exists := false
if err != nil {
logger.WithFields(logrus.Fields{
"DescribeStackOutputError": err,
}).Debug("DescribeStackOutput")
// If the stack doesn't exist, then no worries
if strings.Contains(err.Error(), "does not exist") {
exists = false
} else {
return false, err
}
} else {
exists = true
}
return exists, nil
}
func (r *Run) runUpsert(client *cf.CloudFormation, d tacks.Document, stack string) error {
e := d.Environment
resp, _ := client.DescribeStacks(&cf.DescribeStacksInput{
StackName: aws.String(e.StackName),
})
if len(resp.Stacks) == 0 {
return r.runCreate(client, d, stack)
} else {
return r.runUpdate(client, d, stack)
}
}
// WaitForStackOperationComplete is a blocking, polling based call that
// periodically fetches the stackID set of events and uses the state value
// to determine if an operation is complete
func WaitForStackOperationComplete(stackID string,
pollingMessage string,
awsCloudFormation *cloudformation.CloudFormation,
logger *logrus.Logger) (*WaitForStackOperationCompleteResult, error) {
result := &WaitForStackOperationCompleteResult{}
// Poll for the current stackID state, and
describeStacksInput := &cloudformation.DescribeStacksInput{
StackName: aws.String(stackID),
}
for waitComplete := false; !waitComplete; {
sleepDuration := time.Duration(11+rand.Int31n(13)) * time.Second
time.Sleep(sleepDuration)
describeStacksOutput, err := awsCloudFormation.DescribeStacks(describeStacksInput)
if nil != err {
// TODO - add retry iff we're RateExceeded due to collective access
return nil, err
}
if len(describeStacksOutput.Stacks) <= 0 {
return nil, fmt.Errorf("Failed to enumerate stack info: %v", *describeStacksInput.StackName)
}
result.stackInfo = describeStacksOutput.Stacks[0]
switch *(result.stackInfo).StackStatus {
case cloudformation.StackStatusCreateComplete,
cloudformation.StackStatusUpdateComplete:
result.operationSuccessful = true
waitComplete = true
case
// Include DeleteComplete as new provisions will automatically rollback
cloudformation.StackStatusDeleteComplete,
cloudformation.StackStatusCreateFailed,
cloudformation.StackStatusDeleteFailed,
cloudformation.StackStatusRollbackFailed,
cloudformation.StackStatusRollbackComplete,
cloudformation.StackStatusUpdateRollbackComplete:
result.operationSuccessful = false
waitComplete = true
default:
logger.Info(pollingMessage)
}
}
return result, nil
}
func waitForCompletion(stack string, CloudFormation *cloudformation.CloudFormation, isDeleting bool) (string, error) {
for {
dres, err := CloudFormation.DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(stack),
})
if err != nil {
return "", err
}
err = displayProgress(stack, CloudFormation, isDeleting)
if err != nil {
return "", err
}
if len(dres.Stacks) != 1 {
return "", fmt.Errorf("could not read stack status")
}
switch *dres.Stacks[0].StackStatus {
case "CREATE_COMPLETE":
for _, o := range dres.Stacks[0].Outputs {
if *o.OutputKey == "Dashboard" {
return *o.OutputValue, nil
}
}
stdcli.QOSEventSend("cli-install", distinctID, stdcli.QOSEventProperties{Error: err})
return "", fmt.Errorf("could not install stack, contact [email protected] for assistance")
case "CREATE_FAILED":
stdcli.QOSEventSend("cli-install", distinctID, stdcli.QOSEventProperties{Error: err})
return "", fmt.Errorf("stack creation failed, contact [email protected] for assistance")
case "ROLLBACK_COMPLETE":
stdcli.QOSEventSend("cli-install", distinctID, stdcli.QOSEventProperties{Error: err})
return "", fmt.Errorf("stack creation failed, contact [email protected] for assistance")
case "DELETE_COMPLETE":
stdcli.QOSEventSend("cli-install", distinctID, stdcli.QOSEventProperties{Error: err})
return "", nil
case "DELETE_FAILED":
stdcli.QOSEventSend("cli-install", distinctID, stdcli.QOSEventProperties{Error: err})
return "", fmt.Errorf("stack deletion failed, contact [email protected] for assistance")
}
time.Sleep(2 * time.Second)
}
}
func waitForStackCreateComplete(svc *cloudformation.CloudFormation, stackID string) error {
req := cloudformation.DescribeStacksInput{
StackName: aws.String(stackID),
}
for {
resp, err := svc.DescribeStacks(&req)
if err != nil {
return err
}
if len(resp.Stacks) == 0 {
return fmt.Errorf("stack not found")
}
switch aws.StringValue(resp.Stacks[0].StackStatus) {
case cloudformation.ResourceStatusCreateComplete:
return nil
case cloudformation.ResourceStatusCreateFailed:
return errors.New(aws.StringValue(resp.Stacks[0].StackStatusReason))
}
time.Sleep(3 * time.Second)
}
}
func watch(svc *awscf.CloudFormation, verbose bool, interval int, stackName string) {
req := &awscf.DescribeStacksInput{StackName: aws.String(stackName)}
var maxError int
previousStatus := ""
var err error
var firstLoop bool
for maxError < 3 {
if !firstLoop {
time.Sleep(time.Duration(interval) * time.Second)
} else {
firstLoop = true
}
resp, err := svc.DescribeStacks(req)
if err != nil {
if previousStatus == "DELETE_IN_PROGRESS" {
fmt.Printf("%s Finished\n", time.Now().Format(time.RFC3339))
return
}
fmt.Printf("Error: %s - retrying\n", err)
maxError++
continue
}
for _, stack := range resp.Stacks {
if *stack.StackName == stackName {
if *stack.StackStatus != previousStatus || verbose {
fmt.Printf("%s %s\n", time.Now().Format(time.RFC3339), *stack.StackStatus)
previousStatus = *stack.StackStatus
}
}
if strings.HasSuffix(previousStatus, "COMPLETE") {
fmt.Printf("%s Finished\n", time.Now().Format(time.RFC3339))
return
}
}
}
fmt.Printf("Error: %s - giving up\n", err)
}
func waitForStackUpdateComplete(svc *cloudformation.CloudFormation, stackID string) error {
req := cloudformation.DescribeStacksInput{
StackName: aws.String(stackID),
}
for {
resp, err := svc.DescribeStacks(&req)
if err != nil {
return err
}
if len(resp.Stacks) == 0 {
return fmt.Errorf("stack not found")
}
statusString := aws.StringValue(resp.Stacks[0].StackStatus)
switch statusString {
case cloudformation.ResourceStatusUpdateComplete:
return nil
case cloudformation.ResourceStatusUpdateFailed, cloudformation.StackStatusUpdateRollbackComplete, cloudformation.StackStatusUpdateRollbackFailed:
errMsg := fmt.Sprintf("Stack status: %s : %s", statusString, aws.StringValue(resp.Stacks[0].StackStatusReason))
return errors.New(errMsg)
}
time.Sleep(3 * time.Second)
}
}
[]*cloudformation.Parameter{
&cloudformation.Parameter{
ParameterKey: aws.String("some-key-0"),
ParameterValue: aws.String("some-value-0"),
},
&cloudformation.Parameter{
ParameterKey: aws.String("some-key-1"),
ParameterValue: aws.String("some-value-1"),
},
},
))
})
It("should call the backend method", func() {
client.DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String("some-stack-name"),
})
Expect(fakeBackend.DescribeStacksCall.Receives).NotTo(BeNil())
Expect(fakeBackend.DescribeStacksCall.Receives.StackName).To(Equal(aws.String("some-stack-name")))
})
Context("when the backend succeeds", func() {
It("should return the data in a format parsable by the client library", func() {
fakeBackend.DescribeStacksCall.ReturnsResult = &cloudformation.DescribeStacksOutput{
Stacks: []*cloudformation.Stack{
&cloudformation.Stack{
StackName: aws.String("first stack"),
Outputs: []*cloudformation.Output{
&cloudformation.Output{
OutputKey: aws.String("some-key"),
// describeRackStacks uses credentials to describe CF service, app and rack stacks that belong to the rack name and region
func describeRackStacks(rackName, distinctId string, CF *cloudformation.CloudFormation) (Stacks, error) {
res, err := CF.DescribeStacks(&cloudformation.DescribeStacksInput{})
if err != nil {
return Stacks{}, err
}
apps := []Stack{}
rack := []Stack{}
services := []Stack{}
for _, stack := range res.Stacks {
events := map[string]string{}
outputs := map[string]string{}
tags := map[string]string{}
for _, output := range stack.Outputs {
outputs[*output.OutputKey] = *output.OutputValue
}
for _, tag := range stack.Tags {
tags[*tag.Key] = *tag.Value
}
name := tags["Name"]
if name == "" {
name = *stack.StackName
}
buckets := []string{}
rres, err := CF.DescribeStackResources(&cloudformation.DescribeStackResourcesInput{
StackName: stack.StackId,
})
if err != nil {
return Stacks{}, err
}
for _, resource := range rres.StackResources {
if *resource.ResourceType == "AWS::S3::Bucket" {
if resource.PhysicalResourceId != nil {
buckets = append(buckets, *resource.PhysicalResourceId)
}
}
}
eres, err := CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: stack.StackId,
})
if err != nil {
return Stacks{}, err
}
for _, event := range eres.StackEvents {
if strings.HasSuffix(*event.ResourceStatus, "FAILED") {
events[*event.LogicalResourceId] = *event.ResourceStatusReason
}
}
s := Stack{
Name: name,
StackName: *stack.StackName,
Status: *stack.StackStatus,
Type: tags["Type"],
Buckets: buckets,
Events: events,
Outputs: outputs,
}
// collect stacks that are explicitly related to the rack
if tags["Rack"] == rackName {
switch tags["Type"] {
case "app":
apps = append(apps, s)
case "service":
services = append(services, s)
}
}
// collect stack that is explicitly the rack
if *stack.StackName == rackName && outputs["Dashboard"] != "" {
rack = append(rack, s)
}
}
return Stacks{
Apps: apps,
Rack: rack,
Services: services,
}, nil
}
