// Return the StackEvents for the given StackName/StackID
func stackEvents(stackID string, cfService *cloudformation.CloudFormation) ([]*cloudformation.StackEvent, error) {
var events []*cloudformation.StackEvent
nextToken := ""
for {
params := &cloudformation.DescribeStackEventsInput{
StackName: aws.String(stackID),
}
if len(nextToken) > 0 {
params.NextToken = aws.String(nextToken)
}
resp, err := cfService.DescribeStackEvents(params)
if nil != err {
return nil, err
}
events = append(events, resp.StackEvents...)
if nil == resp.NextToken {
break
} else {
nextToken = *resp.NextToken
}
}
return events, nil
}
// getCloudFormationFailures returns ResourceStatusReason(s)
// of events that should be failures based on regexp match of status
func getCloudFormationFailures(stackName *string, afterTime time.Time,
conn *cloudformation.CloudFormation) ([]string, error) {
var failures []string
// Only catching failures from last 100 events
// Some extra iteration logic via NextToken could be added
// but in reality it's nearly impossible to generate >100
// events by a single stack update
events, err := conn.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: stackName,
})
if err != nil {
return nil, err
}
failRe := regexp.MustCompile("_FAILED$")
rollbackRe := regexp.MustCompile("^ROLLBACK_")
for _, e := range events.StackEvents {
if (failRe.MatchString(*e.ResourceStatus) || rollbackRe.MatchString(*e.ResourceStatus)) &&
e.Timestamp.After(afterTime) && e.ResourceStatusReason != nil {
failures = append(failures, *e.ResourceStatusReason)
}
}
return failures, nil
}
// getLastCfEventTimestamp takes the first event in a list
// of events ordered from the newest to the oldest
// and extracts timestamp from it
// LastUpdatedTime only provides last >successful< updated time
func getLastCfEventTimestamp(stackName string, conn *cloudformation.CloudFormation) (
*time.Time, error) {
output, err := conn.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: aws.String(stackName),
})
if err != nil {
return nil, err
}
return output.StackEvents[0].Timestamp, nil
}
func descStack(svc *awscf.CloudFormation, stackName string) {
input := &awscf.DescribeStackEventsInput{
StackName: aws.String(stackName),
}
resp, err := svc.DescribeStackEvents(input)
if err != nil {
log.Fatal(err)
}
if len(resp.StackEvents) > 0 {
log.Println(awsutil.StringValue(resp.StackEvents[0]))
}
}
func runDescribeStackEevntsQuery(svc *cloudformation.CloudFormation, params *cloudformation.DescribeStackEventsInput) []*cloudformation.StackEvent {
resp, err := svc.DescribeStackEvents(params)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
// Generic AWS error with Code, Message, and original error (if any)
fmt.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
if reqErr, ok := err.(awserr.RequestFailure); ok {
// A service error occurred
fmt.Println(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
}
} else {
// This case should never be hit, the SDK should always return an
// error which satisfies the awserr.Error interface.
fmt.Println(err.Error())
}
}
return resp.StackEvents
}
func displayProgress(stack string, CloudFormation *cloudformation.CloudFormation, isDeleting bool) error {
res, err := CloudFormation.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: aws.String(stack),
})
if err != nil {
return err
}
for _, event := range res.StackEvents {
if events[*event.EventId] == true {
continue
}
events[*event.EventId] = true
// Log all CREATE_FAILED to display and MixPanel
if !isDeleting && *event.ResourceStatus == "CREATE_FAILED" {
msg := fmt.Sprintf("Failed %s: %s", *event.ResourceType, *event.ResourceStatusReason)
fmt.Println(msg)
sendMixpanelEvent("convox-install-error", msg)
}
name := friendlyName(*event.ResourceType)
if name == "" {
continue
}
switch *event.ResourceStatus {
case "CREATE_IN_PROGRESS":
case "CREATE_COMPLETE":
if !isDeleting {
id := *event.PhysicalResourceId
if strings.HasPrefix(id, "arn:") {
id = *event.LogicalResourceId
}
fmt.Printf("Created %s: %s\n", name, id)
}
case "CREATE_FAILED":
case "DELETE_IN_PROGRESS":
case "DELETE_COMPLETE":
id := *event.PhysicalResourceId
if strings.HasPrefix(id, "arn:") {
id = *event.LogicalResourceId
}
fmt.Printf("Deleted %s: %s\n", name, id)
case "DELETE_SKIPPED":
id := *event.PhysicalResourceId
if strings.HasPrefix(id, "arn:") {
id = *event.LogicalResourceId
}
fmt.Printf("Skipped %s: %s\n", name, id)
case "DELETE_FAILED":
return fmt.Errorf("stack deletion failed")
case "ROLLBACK_IN_PROGRESS", "ROLLBACK_COMPLETE":
case "UPDATE_IN_PROGRESS", "UPDATE_COMPLETE", "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", "UPDATE_FAILED", "UPDATE_ROLLBACK_IN_PROGRESS", "UPDATE_ROLLBACK_COMPLETE", "UPDATE_ROLLBACK_FAILED":
default:
return fmt.Errorf("Unhandled status: %s\n", *event.ResourceStatus)
}
}
return nil
}
// describeRackStacks uses credentials to describe CF service, app and rack stacks that belong to the rack name and region
func describeRackStacks(rackName, distinctId string, CF *cloudformation.CloudFormation) (Stacks, error) {
res, err := CF.DescribeStacks(&cloudformation.DescribeStacksInput{})
if err != nil {
return Stacks{}, err
}
apps := []Stack{}
rack := []Stack{}
services := []Stack{}
for _, stack := range res.Stacks {
events := map[string]string{}
outputs := map[string]string{}
tags := map[string]string{}
for _, output := range stack.Outputs {
outputs[*output.OutputKey] = *output.OutputValue
}
for _, tag := range stack.Tags {
tags[*tag.Key] = *tag.Value
}
name := tags["Name"]
if name == "" {
name = *stack.StackName
}
buckets := []string{}
rres, err := CF.DescribeStackResources(&cloudformation.DescribeStackResourcesInput{
StackName: stack.StackId,
})
if err != nil {
return Stacks{}, err
}
for _, resource := range rres.StackResources {
if *resource.ResourceType == "AWS::S3::Bucket" {
if resource.PhysicalResourceId != nil {
buckets = append(buckets, *resource.PhysicalResourceId)
}
}
}
eres, err := CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{
StackName: stack.StackId,
})
if err != nil {
return Stacks{}, err
}
for _, event := range eres.StackEvents {
if strings.HasSuffix(*event.ResourceStatus, "FAILED") {
events[*event.LogicalResourceId] = *event.ResourceStatusReason
}
}
s := Stack{
Name: name,
StackName: *stack.StackName,
Status: *stack.StackStatus,
Type: tags["Type"],
Buckets: buckets,
Events: events,
Outputs: outputs,
}
// collect stacks that are explicitly related to the rack
if tags["Rack"] == rackName {
switch tags["Type"] {
case "app":
apps = append(apps, s)
case "service":
services = append(services, s)
}
}
// collect stack that is explicitly the rack
if *stack.StackName == rackName && outputs["Dashboard"] != "" {
rack = append(rack, s)
}
}
return Stacks{
Apps: apps,
Rack: rack,
Services: services,
}, nil
}