func dequeueMessage() ([]Message, error) { req := &sqs.ReceiveMessageInput{ MaxNumberOfMessages: aws.Int64(10), QueueUrl: aws.String(MessageQueueUrl), WaitTimeSeconds: aws.Int64(10), } res, err := SQS().ReceiveMessage(req) if err != nil { return nil, err } messages := make([]Message, len(res.Messages)) var message Message for i, m := range res.Messages { err = json.Unmarshal([]byte(*m.Body), &message) if err != nil { return nil, err } message.MessageID = m.MessageId message.ReceiptHandle = m.ReceiptHandle messages[i] = message } return messages, nil }
func ECSServiceCreate(req Request) (string, map[string]string, error) { count, err := strconv.Atoi(req.ResourceProperties["DesiredCount"].(string)) if err != nil { return "", nil, err } r := &ecs.CreateServiceInput{ Cluster: aws.String(req.ResourceProperties["Cluster"].(string)), DesiredCount: aws.Int64(int64(count)), ServiceName: aws.String(req.ResourceProperties["Name"].(string) + "-" + generateId("S", 10)), TaskDefinition: aws.String(req.ResourceProperties["TaskDefinition"].(string)), } balancers := req.ResourceProperties["LoadBalancers"].([]interface{}) if len(balancers) > 0 { r.Role = aws.String(req.ResourceProperties["Role"].(string)) } for _, balancer := range balancers { parts := strings.SplitN(balancer.(string), ":", 3) if len(parts) != 3 { return "", nil, fmt.Errorf("invalid load balancer specification: %s", balancer.(string)) } name := parts[0] ps := parts[1] port, _ := strconv.Atoi(parts[2]) r.LoadBalancers = append(r.LoadBalancers, &ecs.LoadBalancer{ LoadBalancerName: aws.String(name), ContainerName: aws.String(ps), ContainerPort: aws.Int64(int64(port)), }) break } res, err := ECS(req).CreateService(r) if err != nil { return "", nil, err } return *res.Service.ServiceArn, nil, nil }
func ListReleases(app string) (Releases, error) { req := &dynamodb.QueryInput{ KeyConditions: map[string]*dynamodb.Condition{ "app": &dynamodb.Condition{ AttributeValueList: []*dynamodb.AttributeValue{ &dynamodb.AttributeValue{S: aws.String(app)}, }, ComparisonOperator: aws.String("EQ"), }, }, IndexName: aws.String("app.created"), Limit: aws.Int64(20), ScanIndexForward: aws.Bool(false), TableName: aws.String(releasesTable(app)), } res, err := DynamoDB().Query(req) if err != nil { return nil, err } releases := make(Releases, len(res.Items)) for i, item := range res.Items { releases[i] = *releaseFromItem(item) } return releases, nil }
// Retrieve generates a new set of temporary credentials using STS. func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) { // Apply defaults where parameters are not set. if p.Client == nil { p.Client = sts.New(nil) } if p.RoleSessionName == "" { // Try to work out a role name that will hopefully end up unique. p.RoleSessionName = fmt.Sprintf("%d", time.Now().UTC().UnixNano()) } if p.Duration == 0 { // Expire as often as AWS permits. p.Duration = 15 * time.Minute } roleOutput, err := p.Client.AssumeRole(&sts.AssumeRoleInput{ DurationSeconds: aws.Int64(int64(p.Duration / time.Second)), RoleArn: aws.String(p.RoleARN), RoleSessionName: aws.String(p.RoleSessionName), ExternalId: p.ExternalID, }) if err != nil { return credentials.Value{}, err } // We will proactively generate new credentials before they expire. p.SetExpiration(*roleOutput.Credentials.Expiration, p.ExpiryWindow) return credentials.Value{ AccessKeyID: *roleOutput.Credentials.AccessKeyId, SecretAccessKey: *roleOutput.Credentials.SecretAccessKey, SessionToken: *roleOutput.Credentials.SessionToken, }, nil }
func (a *App) RunDetached(process, command string) error { resources := a.Resources() req := &ecs.RunTaskInput{ Cluster: aws.String(os.Getenv("CLUSTER")), Count: aws.Int64(1), TaskDefinition: aws.String(resources[UpperName(process)+"ECSTaskDefinition"].Id), } if command != "" { req.Overrides = &ecs.TaskOverride{ ContainerOverrides: []*ecs.ContainerOverride{ &ecs.ContainerOverride{ Name: aws.String(process), Command: []*string{ aws.String("sh"), aws.String("-c"), aws.String(command), }, }, }, } } _, err := ECS().RunTask(req) if err != nil { return err } return nil }
func LambdaFunctionCreate(req Request) (string, error) { bres, err := http.Get(req.ResourceProperties["ZipFile"].(string)) if err != nil { return "", err } defer bres.Body.Close() body, err := ioutil.ReadAll(bres.Body) memory := 128 timeout := 5 if m, ok := req.ResourceProperties["Memory"].(string); ok && m != "" { memory, _ = strconv.Atoi(m) } if t, ok := req.ResourceProperties["Timeout"].(string); ok && t != "" { timeout, _ = strconv.Atoi(t) } role := fmt.Sprintf("arn:aws:iam::%s:role/%s", req.ResourceProperties["AccountId"].(string), req.ResourceProperties["Role"].(string)) res, err := Lambda(req).CreateFunction(&lambda.CreateFunctionInput{ FunctionName: aws.String(req.ResourceProperties["Name"].(string)), Handler: aws.String(req.ResourceProperties["Handler"].(string)), MemorySize: aws.Int64(int64(memory)), Timeout: aws.Int64(int64(timeout)), Role: aws.String(role), Runtime: aws.String(req.ResourceProperties["Runtime"].(string)), Code: &lambda.FunctionCode{ ZipFile: body, }, }) if err != nil { return "", err } return *res.FunctionArn, nil }
func S3Put(bucket, key string, data []byte, public bool) error { req := &s3.PutObjectInput{ Body: bytes.NewReader(data), Bucket: aws.String(bucket), ContentLength: aws.Int64(int64(len(data))), Key: aws.String(key), } if public { req.ACL = aws.String("public-read") } _, err := S3().PutObject(req) return err }
func ECSServiceUpdate(req Request) (string, map[string]string, error) { count, _ := strconv.Atoi(req.ResourceProperties["DesiredCount"].(string)) // arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY parts := strings.Split(req.PhysicalResourceId, "/") name := parts[1] res, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{ Cluster: aws.String(req.ResourceProperties["Cluster"].(string)), Service: aws.String(name), DesiredCount: aws.Int64(int64(count)), TaskDefinition: aws.String(req.ResourceProperties["TaskDefinition"].(string)), }) if err != nil { return "", nil, err } return *res.Service.ServiceArn, nil, nil }
func ECSServiceDelete(req Request) (string, map[string]string, error) { cluster := req.ResourceProperties["Cluster"].(string) // arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY parts := strings.Split(req.PhysicalResourceId, "/") name := parts[1] _, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{ Cluster: aws.String(cluster), Service: aws.String(name), DesiredCount: aws.Int64(0), }) // go ahead and mark the delete good if the service is not found if ae, ok := err.(awserr.Error); ok { if ae.Code() == "ServiceNotFoundException" { return req.PhysicalResourceId, nil, nil } } // TODO let the cloudformation finish thinking this deleted // but take note so we can figure out why if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err) return req.PhysicalResourceId, nil, nil } _, err = ECS(req).DeleteService(&ecs.DeleteServiceInput{ Cluster: aws.String(cluster), Service: aws.String(name), }) // TODO let the cloudformation finish thinking this deleted // but take note so we can figure out why if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err) return req.PhysicalResourceId, nil, nil } return req.PhysicalResourceId, nil, nil }
func getMetric(metric string, dimensions []*cloudwatch.Dimension, span, precision int64) ([]*cloudwatch.Datapoint, error) { req := &cloudwatch.GetMetricStatisticsInput{ Dimensions: dimensions, EndTime: aws.Time(time.Now()), MetricName: aws.String(metric), Namespace: aws.String("Convox"), Period: aws.Int64(precision * 60), StartTime: aws.Time(time.Now().Add(time.Duration(-1*span) * time.Minute)), Statistics: []*string{aws.String("Average")}, } res, err := CloudWatch().GetMetricStatistics(req) if err != nil { // TODO log error fmt.Printf("error fetching metrics: %s\n", err) return []*cloudwatch.Datapoint{}, nil } return res.Datapoints, nil }
func S3PutFile(bucket, key string, f io.ReadSeeker, public bool) error { // seek to end of f to determine length, then seek back to beginning for upload l, err := f.Seek(0, 2) if err != nil { return err } _, err = f.Seek(0, 0) if err != nil { return err } req := &s3.PutObjectInput{ Body: f, Bucket: aws.String(bucket), ContentLength: aws.Int64(l), Key: aws.String(key), } if public { req.ACL = aws.String("public-read") } _, err = S3().PutObject(req) if err != nil { return err } // seek back to beginning in case something else needs to read f _, err = f.Seek(0, 0) return err }
func ECSTaskDefinitionCreate(req Request) (string, map[string]string, error) { // return "", fmt.Errorf("fail") tasks := req.ResourceProperties["Tasks"].([]interface{}) r := &ecs.RegisterTaskDefinitionInput{ Family: aws.String(req.ResourceProperties["Name"].(string)), } // get environment from S3 URL // 'Environment' is a CloudFormation Template Property that references 'Environment' CF Parameter with S3 URL // S3 body may be encrypted with KMS key var env models.Environment if envUrl, ok := req.ResourceProperties["Environment"].(string); ok && envUrl != "" { res, err := http.Get(envUrl) if err != nil { return "", nil, err } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if key, ok := req.ResourceProperties["Key"].(string); ok && key != "" { cr := crypt.New(*Region(&req), os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")) cr.AwsToken = os.Getenv("AWS_SESSION_TOKEN") dec, err := cr.Decrypt(key, data) if err != nil { return "", nil, err } data = dec } env = models.LoadEnvironment(data) } r.ContainerDefinitions = make([]*ecs.ContainerDefinition, len(tasks)) for i, itask := range tasks { task := itask.(map[string]interface{}) memory, _ := strconv.Atoi(task["Memory"].(string)) r.ContainerDefinitions[i] = &ecs.ContainerDefinition{ Name: aws.String(task["Name"].(string)), Essential: aws.Bool(true), Image: aws.String(task["Image"].(string)), Memory: aws.Int64(int64(memory)), } if command, ok := task["Command"].(string); ok && command != "" { r.ContainerDefinitions[i].Command = []*string{aws.String("sh"), aws.String("-c"), aws.String(command)} } // set Task environment from CF Tasks[].Environment key/values // These key/values are read from the app manifest environment hash if oenv, ok := task["Environment"].(map[string]interface{}); ok { for key, val := range oenv { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String(key), Value: aws.String(val.(string)), }) } } // set Task environment from decrypted S3 URL body of key/values // These key/values take precident over the above environment for key, val := range env { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String(key), Value: aws.String(val), }) } // set Release value in Task environment if release, ok := req.ResourceProperties["Release"].(string); ok { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String("RELEASE"), Value: aws.String(release), }) } // set links if links, ok := task["Links"].([]interface{}); ok { r.ContainerDefinitions[i].Links = make([]*string, len(links)) for j, link := range links { r.ContainerDefinitions[i].Links[j] = aws.String(link.(string)) } } // set portmappings if ports, ok := task["PortMappings"].([]interface{}); ok { r.ContainerDefinitions[i].PortMappings = make([]*ecs.PortMapping, len(ports)) for j, port := range ports { parts := strings.Split(port.(string), ":") host, _ := strconv.Atoi(parts[0]) container, _ := strconv.Atoi(parts[1]) r.ContainerDefinitions[i].PortMappings[j] = &ecs.PortMapping{ ContainerPort: aws.Int64(int64(container)), HostPort: aws.Int64(int64(host)), } } } // set volumes if volumes, ok := task["Volumes"].([]interface{}); ok { for j, volume := range volumes { name := fmt.Sprintf("%s-%d-%d", task["Name"].(string), i, j) parts := strings.Split(volume.(string), ":") r.Volumes = append(r.Volumes, &ecs.Volume{ Name: aws.String(name), Host: &ecs.HostVolumeProperties{ SourcePath: aws.String(parts[0]), }, }) r.ContainerDefinitions[i].MountPoints = append(r.ContainerDefinitions[i].MountPoints, &ecs.MountPoint{ SourceVolume: aws.String(name), ContainerPath: aws.String(parts[1]), ReadOnly: aws.Bool(false), }) } } } res, err := ECS(req).RegisterTaskDefinition(r) if err != nil { return "", nil, err } return *res.TaskDefinition.TaskDefinitionArn, nil, nil }
func subscribeRDS(prefix, id string, output chan []byte, quit chan bool) { // Get latest log file details via pagination tokens details := rds.DescribeDBLogFilesDetails{} marker := "" for { params := &rds.DescribeDBLogFilesInput{ DBInstanceIdentifier: aws.String(id), MaxRecords: aws.Int64(100), } if marker != "" { params.Marker = aws.String(marker) } res, err := RDS().DescribeDBLogFiles(params) if err != nil { panic(err) } if res.Marker == nil { files := res.DescribeDBLogFiles details = *files[len(files)-1] break } marker = *res.Marker } // Get last 50 log lines params := &rds.DownloadDBLogFilePortionInput{ DBInstanceIdentifier: aws.String(id), LogFileName: aws.String(*details.LogFileName), NumberOfLines: aws.Int64(50), } res, err := RDS().DownloadDBLogFilePortion(params) if err != nil { panic(err) } output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData)) params.Marker = aws.String(*res.Marker) for { select { case <-quit: fmt.Println("quitting") return default: res, err := RDS().DownloadDBLogFilePortion(params) if err != nil { panic(err) } if *params.Marker != *res.Marker { params.Marker = aws.String(*res.Marker) output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData)) } time.Sleep(1000 * time.Millisecond) } } }