func StartHeartbeat() {
log := logger.New("ns=heartbeat")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
helpers.TrackEvent("kernel-heartbeat", "")
for _ = range time.Tick(1 * time.Hour) {
helpers.TrackEvent("kernel-heartbeat", "")
}
}
func (a *App) Delete() error {
helpers.TrackEvent("kernel-app-delete-start", nil)
_, err := CloudFormation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(a.StackName())})
if err != nil {
helpers.TrackEvent("kernel-app-delete-error", nil)
return err
}
go a.Cleanup()
NotifySuccess("app:delete", map[string]string{"name": a.Name})
return nil
}
// AppDelete deletes an app
func (p *AWSProvider) AppDelete(name string) error {
app, err := p.AppGet(name)
if err != nil {
return err
}
services, err := p.ServiceList()
if err != nil {
return err
}
for _, s := range services {
s.Apps, err = p.serviceApps(s)
if err != nil {
return err
}
for _, a := range s.Apps {
if a.Name == name {
return fmt.Errorf("app is linked to %s service", s.Name)
}
}
}
_, err = p.cloudformation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(app.StackName())})
if err != nil {
helpers.TrackEvent("kernel-app-delete-error", nil)
return err
}
go p.cleanup(app)
return nil
}
func (r *System) Save() error {
rack := os.Getenv("RACK")
app, err := GetApp(rack)
if err != nil {
return err
}
mac, err := maxAppConcurrency()
// dont scale the rack below the max concurrency plus one
// see formation.go for more details
if err == nil && r.Count < (mac+1) {
return fmt.Errorf("max process concurrency is %d, can't scale rack below %d instances", mac, mac+1)
}
params := map[string]string{
"InstanceCount": strconv.Itoa(r.Count),
"InstanceType": r.Type,
"Version": r.Version,
}
// Report cluster size change
helpers.TrackEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", r.Count, r.Type))
template := fmt.Sprintf("https://convox.s3.amazonaws.com/release/%s/formation.json", r.Version)
return app.UpdateParamsAndTemplate(params, template)
}
func (a *App) Delete() error {
helpers.TrackEvent("kernel-app-delete-start", "")
name := a.Name
_, err := CloudFormation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(name)})
if err != nil {
helpers.TrackEvent("kernel-app-delete-error", "")
return err
}
go a.Cleanup()
helpers.TrackEvent("kernel-app-delete-success", "")
return nil
}
func (a *App) Delete() error {
helpers.TrackEvent("kernel-app-delete-start", nil)
err := Provider().AppDelete(a.Name)
if err != nil {
return err
}
NotifySuccess("app:delete", map[string]string{"name": a.Name})
return nil
}
// EventSend publishes an important message out to the world.
//
// On AWS messages are published to SNS. The Rack has an HTTP endpoint that is an SNS
// subscription, and when a message is delivered forwards them to all configured
// webhook services.
//
// Often the Rack has a Console webhook which facilitates forwarding events
// to Slack with additional formatting and filtering.
//
// Because these are important system events, they are also published to Segment
// for operational metrics.
func (p *AWSProvider) EventSend(e *structs.Event, err error) error {
// log := logger.New("ns=kernel")
e.Status = "success"
e.Timestamp = time.Now().UTC()
if p.IsTest() {
e.Timestamp = time.Time{}
}
if err != nil {
e.Data["message"] = err.Error()
e.Status = "error"
}
msg, err := json.Marshal(e)
if err != nil {
// helpers.Error(log, err) // report internal errors to Rollbar
return err
}
fmt.Printf("aws EventSend msg=%q\n", msg)
// Publish Event to SNS
_, err = p.sns().Publish(&sns.PublishInput{
Message: aws.String(string(msg)), // Required
Subject: aws.String(e.Action),
TargetArn: aws.String(p.NotificationTopic),
})
if err != nil {
// helpers.Error(log, err) // report internal errors to Rollbar
return err
}
// log.At("EventSend").Log("message-id=%q", *resp.MessageId)
// report event to Segment
params := map[string]interface{}{
"action": e.Action,
"status": e.Status,
}
for k, v := range e.Data {
params[k] = v
}
helpers.TrackEvent("event", params)
return nil
}
// AppDelete deletes an app
func (p *AWSProvider) AppDelete(name string) error {
app, err := p.AppGet(name)
if err != nil {
return err
}
_, err = p.cloudformation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(app.StackName())})
if err != nil {
helpers.TrackEvent("kernel-app-delete-error", nil)
return err
}
go p.cleanup(app)
return nil
}
// EventSend publishes an important message out to the world.
//
// On AWS messages are published to SNS. The Rack has an HTTP endpoint that is an SNS
// subscription, and when a message is delivered forwards them to all configured
// webhook services.
//
// Often the Rack has a Console webhook which facilitates forwarding events
// to Slack with additional formatting and filtering.
//
// Because these are important system events, they are also published to Segment
// for operational metrics.
func (p *AWSProvider) EventSend(e *structs.Event, err error) error {
e.Status = "success"
e.Timestamp = time.Now().UTC()
if p.IsTest() {
e.Timestamp = time.Time{}
}
if err != nil {
e.Data["message"] = err.Error()
e.Status = "error"
}
msg, err := json.Marshal(e)
if err != nil {
return err
}
// Publish Event to SNS
_, err = p.sns().Publish(&sns.PublishInput{
Message: aws.String(string(msg)), // Required
Subject: aws.String(e.Action),
TargetArn: aws.String(p.NotificationTopic),
})
if err != nil {
return err
}
// report event to Segment
params := map[string]interface{}{
"action": e.Action,
"status": e.Status,
}
for k, v := range e.Data {
params[k] = v
}
helpers.TrackEvent("event", params)
return nil
}
func heartbeat() {
system, err := models.GetSystem()
if err != nil {
log.Error(err)
return
}
apps, err := models.ListApps()
if err != nil {
log.Error(err)
return
}
helpers.TrackEvent("kernel-heartbeat", map[string]interface{}{
"app_count": len(apps),
"instance_count": system.Count,
"instance_type": system.Type,
"region": os.Getenv("AWS_REGION"),
"version": system.Version,
})
}
func (a *App) Create() error {
helpers.TrackEvent("kernel-app-create-start", nil)
if !regexValidAppName.MatchString(a.Name) {
return fmt.Errorf("app name can contain only alphanumeric characters and dashes and must be between 4 and 30 characters")
}
formation, err := a.Formation()
if err != nil {
helpers.TrackEvent("kernel-app-create-error", nil)
return err
}
params := map[string]string{
"Cluster": os.Getenv("CLUSTER"),
"Subnets": os.Getenv("SUBNETS"),
"Version": os.Getenv("RELEASE"),
"VPC": os.Getenv("VPC"),
}
if os.Getenv("ENCRYPTION_KEY") != "" {
params["Key"] = os.Getenv("ENCRYPTION_KEY")
}
tags := map[string]string{
"Rack": os.Getenv("RACK"),
"System": "convox",
"Type": "app",
}
req := &cloudformation.CreateStackInput{
Capabilities: []*string{aws.String("CAPABILITY_IAM")},
StackName: aws.String(a.Name),
TemplateBody: aws.String(formation),
}
for key, value := range params {
req.Parameters = append(req.Parameters, &cloudformation.Parameter{
ParameterKey: aws.String(key),
ParameterValue: aws.String(value),
})
}
for key, value := range tags {
req.Tags = append(req.Tags, &cloudformation.Tag{
Key: aws.String(key),
Value: aws.String(value),
})
}
_, err = CloudFormation().CreateStack(req)
if err != nil {
helpers.TrackEvent("kernel-app-create-error", nil)
return err
}
helpers.TrackEvent("kernel-app-create-success", nil)
NotifySuccess("app:create", map[string]string{"name": a.Name})
return nil
}
func (a *App) Cleanup() error {
err := cleanupBucket(a.Outputs["Settings"])
if err != nil {
return err
}
builds, err := ListBuilds(a.Name)
if err != nil {
return err
}
for _, build := range builds {
go cleanupBuild(build)
}
releases, err := ListReleases(a.Name)
if err != nil {
return err
}
for _, release := range releases {
go cleanupRelease(release)
}
// monitor and stack deletion state for up to 10 minutes
// retry once if DELETE_FAILED to automate around transient errors
// send delete success event only when stack is gone
shouldRetry := true
for i := 0; i < 60; i++ {
res, err := CloudFormation().DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(a.StackName()),
})
// return when stack is not found indicating successful delete
if ae, ok := err.(awserr.Error); ok {
if ae.Code() == "ValidationError" {
helpers.TrackEvent("kernel-app-delete-success", nil)
// Last ditch effort to remove the empty bucket CF leaves behind.
_, err := S3().DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(a.Outputs["Settings"])})
if err != nil {
fmt.Printf("error: %s\n", err)
}
return nil
}
}
if err == nil && len(res.Stacks) == 1 && shouldRetry {
// if delete failed, issue one more delete stack and return
s := res.Stacks[0]
if *s.StackStatus == "DELETE_FAILED" {
helpers.TrackEvent("kernel-app-delete-retry", nil)
_, err := CloudFormation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(a.StackName())})
if err != nil {
helpers.TrackEvent("kernel-app-delete-retry-error", nil)
} else {
shouldRetry = false
}
}
}
time.Sleep(10 * time.Second)
}
return nil
}
func (a *App) Create() error {
helpers.TrackEvent("kernel-app-create-start", nil)
if !regexValidAppName.MatchString(a.Name) {
return fmt.Errorf("app name can contain only alphanumeric characters, dashes and must be between 4 and 30 characters")
}
m := manifest.Manifest{
Services: make(map[string]manifest.Service),
}
formation, err := a.Formation(m)
if err != nil {
helpers.TrackEvent("kernel-app-create-error", nil)
return err
}
// SubnetsPrivate is a List<AWS::EC2::Subnet::Id> and can not be empty
// So reuse SUBNETS if SUBNETS_PRIVATE is not set
subnetsPrivate := os.Getenv("SUBNETS_PRIVATE")
if subnetsPrivate == "" {
subnetsPrivate = os.Getenv("SUBNETS")
}
params := map[string]string{
"Cluster": os.Getenv("CLUSTER"),
"Internal": os.Getenv("INTERNAL"),
"Private": os.Getenv("PRIVATE"),
"Subnets": os.Getenv("SUBNETS"),
"SubnetsPrivate": subnetsPrivate,
"Version": os.Getenv("RELEASE"),
"VPC": os.Getenv("VPC"),
"VPCCIDR": os.Getenv("VPCCIDR"),
}
if os.Getenv("ENCRYPTION_KEY") != "" {
params["Key"] = os.Getenv("ENCRYPTION_KEY")
}
tags := map[string]string{
"Rack": os.Getenv("RACK"),
"System": "convox",
"Type": "app",
"Name": a.Name,
}
req := &cloudformation.CreateStackInput{
Capabilities: []*string{aws.String("CAPABILITY_IAM")},
StackName: aws.String(a.StackName()),
TemplateBody: aws.String(formation),
}
for key, value := range params {
req.Parameters = append(req.Parameters, &cloudformation.Parameter{
ParameterKey: aws.String(key),
ParameterValue: aws.String(value),
})
}
for key, value := range tags {
req.Tags = append(req.Tags, &cloudformation.Tag{
Key: aws.String(key),
Value: aws.String(value),
})
}
_, err = CloudFormation().CreateStack(req)
if err != nil {
helpers.TrackEvent("kernel-app-create-error", nil)
return err
}
helpers.TrackEvent("kernel-app-create-success", nil)
NotifySuccess("app:create", map[string]string{"name": a.Name})
return nil
}
func StartCluster() {
var log = logger.New("ns=cluster_monitor")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
// Report cluster size one time on start
system, err := models.GetSystem()
if err != nil {
log.Error(err)
} else {
helpers.TrackEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", system.Count, system.Type))
}
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
instances := Instances{}
err := instances.describeASG()
if err != nil {
log.Error(err)
continue
}
err = instances.describeECS()
if err != nil {
log.Error(err)
continue
}
// TODO: Add an instances.testDocker() call to the mission critical path
// Test if ASG Instance is registered and connected in ECS cluster
for _, i := range instances {
if !i.ASG {
// TODO: Rogue instance?! Terminate?
continue
}
if !i.ECS {
// Not registered or not connected => set Unhealthy
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceId: aws.String(i.Id),
ShouldRespectGracePeriod: aws.Bool(true),
},
)
i.Unhealthy = true
if err != nil {
log.Error(err)
continue
}
}
}
log.Log(instances.log())
}
}
// cleanup deletes AWS resources that aren't handled by the CloudFormation during stack deletion.
func (p *AWSProvider) cleanup(app *structs.App) error {
err := p.deleteBucket(app.Outputs["Settings"])
if err != nil {
fmt.Printf("fn=cleanup level=error msg=\"%s\"", err)
return err
}
err = p.buildsDeleteAll(app)
if err != nil {
fmt.Printf("fn=cleanup level=error msg=\"%s\"", err)
return err
}
_, err = p.ecr().DeleteRepository(&ecr.DeleteRepositoryInput{
RepositoryName: aws.String(app.Outputs["RegistryRepository"]),
Force: aws.Bool(true),
})
if err != nil {
fmt.Printf("fn=cleanup level=error msg=\"error deleting ecr repo: %s\"", err)
}
err = p.releaseDeleteAll(app.Name)
if err != nil {
fmt.Printf("fn=cleanup level=error msg=\"%s\"", err)
return err
}
// monitor and stack deletion state for up to 10 minutes
// retry once if DELETE_FAILED to automate around transient errors
// send delete success event only when stack is gone
shouldRetry := true
for i := 0; i < 60; i++ {
res, err := p.cloudformation().DescribeStacks(&cloudformation.DescribeStacksInput{
StackName: aws.String(app.StackName()),
})
// return when stack is not found indicating successful delete
if ae, ok := err.(awserr.Error); ok {
if ae.Code() == "ValidationError" { // Error indicates stack wasn't found, hence deleted.
helpers.TrackEvent("kernel-app-delete-success", nil)
// Last ditch effort to remove the empty bucket CF leaves behind.
_, err := p.s3().DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(app.Outputs["Settings"])})
if err != nil {
fmt.Printf("last ditch effort bucket error: %s\n", err)
}
return nil
}
}
if err == nil && len(res.Stacks) == 1 && shouldRetry {
// if delete failed, issue one more delete stack and return
s := res.Stacks[0]
if *s.StackStatus == "DELETE_FAILED" {
helpers.TrackEvent("kernel-app-delete-retry", nil)
_, err := p.cloudformation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(app.StackName())})
if err != nil {
helpers.TrackEvent("kernel-app-delete-retry-error", nil)
} else {
shouldRetry = false
}
}
}
time.Sleep(10 * time.Second)
}
return nil
}