func dequeueMessage() ([]Message, error) {
req := &sqs.ReceiveMessageInput{
MaxNumberOfMessages: aws.Int64(10),
QueueUrl: aws.String(MessageQueueUrl),
WaitTimeSeconds: aws.Int64(10),
}
res, err := SQS().ReceiveMessage(req)
if err != nil {
return nil, err
}
messages := make([]Message, len(res.Messages))
var message Message
for i, m := range res.Messages {
err = json.Unmarshal([]byte(*m.Body), &message)
if err != nil {
return nil, err
}
message.MessageID = m.MessageId
message.ReceiptHandle = m.ReceiptHandle
messages[i] = message
}
return messages, nil
}
func ECSServiceCreate(req Request) (string, map[string]string, error) {
count, err := strconv.Atoi(req.ResourceProperties["DesiredCount"].(string))
if err != nil {
return "", nil, err
}
r := &ecs.CreateServiceInput{
Cluster: aws.String(req.ResourceProperties["Cluster"].(string)),
DesiredCount: aws.Int64(int64(count)),
ServiceName: aws.String(req.ResourceProperties["Name"].(string) + "-" + generateId("S", 10)),
TaskDefinition: aws.String(req.ResourceProperties["TaskDefinition"].(string)),
}
balancers := req.ResourceProperties["LoadBalancers"].([]interface{})
if len(balancers) > 0 {
r.Role = aws.String(req.ResourceProperties["Role"].(string))
}
for _, balancer := range balancers {
parts := strings.SplitN(balancer.(string), ":", 3)
if len(parts) != 3 {
return "", nil, fmt.Errorf("invalid load balancer specification: %s", balancer.(string))
}
name := parts[0]
ps := parts[1]
port, _ := strconv.Atoi(parts[2])
r.LoadBalancers = append(r.LoadBalancers, &ecs.LoadBalancer{
LoadBalancerName: aws.String(name),
ContainerName: aws.String(ps),
ContainerPort: aws.Int64(int64(port)),
})
break
}
res, err := ECS(req).CreateService(r)
if err != nil {
return "", nil, err
}
return *res.Service.ServiceArn, nil, nil
}
func ListReleases(app string) (Releases, error) {
req := &dynamodb.QueryInput{
KeyConditions: map[string]*dynamodb.Condition{
"app": &dynamodb.Condition{
AttributeValueList: []*dynamodb.AttributeValue{
&dynamodb.AttributeValue{S: aws.String(app)},
},
ComparisonOperator: aws.String("EQ"),
},
},
IndexName: aws.String("app.created"),
Limit: aws.Int64(20),
ScanIndexForward: aws.Bool(false),
TableName: aws.String(releasesTable(app)),
}
res, err := DynamoDB().Query(req)
if err != nil {
return nil, err
}
releases := make(Releases, len(res.Items))
for i, item := range res.Items {
releases[i] = *releaseFromItem(item)
}
return releases, nil
}
// Retrieve generates a new set of temporary credentials using STS.
func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
// Apply defaults where parameters are not set.
if p.Client == nil {
p.Client = sts.New(nil)
}
if p.RoleSessionName == "" {
// Try to work out a role name that will hopefully end up unique.
p.RoleSessionName = fmt.Sprintf("%d", time.Now().UTC().UnixNano())
}
if p.Duration == 0 {
// Expire as often as AWS permits.
p.Duration = 15 * time.Minute
}
roleOutput, err := p.Client.AssumeRole(&sts.AssumeRoleInput{
DurationSeconds: aws.Int64(int64(p.Duration / time.Second)),
RoleArn: aws.String(p.RoleARN),
RoleSessionName: aws.String(p.RoleSessionName),
ExternalId: p.ExternalID,
})
if err != nil {
return credentials.Value{}, err
}
// We will proactively generate new credentials before they expire.
p.SetExpiration(*roleOutput.Credentials.Expiration, p.ExpiryWindow)
return credentials.Value{
AccessKeyID: *roleOutput.Credentials.AccessKeyId,
SecretAccessKey: *roleOutput.Credentials.SecretAccessKey,
SessionToken: *roleOutput.Credentials.SessionToken,
}, nil
}
func (a *App) RunDetached(process, command string) error {
resources := a.Resources()
req := &ecs.RunTaskInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
Count: aws.Int64(1),
TaskDefinition: aws.String(resources[UpperName(process)+"ECSTaskDefinition"].Id),
}
if command != "" {
req.Overrides = &ecs.TaskOverride{
ContainerOverrides: []*ecs.ContainerOverride{
&ecs.ContainerOverride{
Name: aws.String(process),
Command: []*string{
aws.String("sh"),
aws.String("-c"),
aws.String(command),
},
},
},
}
}
_, err := ECS().RunTask(req)
if err != nil {
return err
}
return nil
}
func LambdaFunctionCreate(req Request) (string, error) {
bres, err := http.Get(req.ResourceProperties["ZipFile"].(string))
if err != nil {
return "", err
}
defer bres.Body.Close()
body, err := ioutil.ReadAll(bres.Body)
memory := 128
timeout := 5
if m, ok := req.ResourceProperties["Memory"].(string); ok && m != "" {
memory, _ = strconv.Atoi(m)
}
if t, ok := req.ResourceProperties["Timeout"].(string); ok && t != "" {
timeout, _ = strconv.Atoi(t)
}
role := fmt.Sprintf("arn:aws:iam::%s:role/%s", req.ResourceProperties["AccountId"].(string), req.ResourceProperties["Role"].(string))
res, err := Lambda(req).CreateFunction(&lambda.CreateFunctionInput{
FunctionName: aws.String(req.ResourceProperties["Name"].(string)),
Handler: aws.String(req.ResourceProperties["Handler"].(string)),
MemorySize: aws.Int64(int64(memory)),
Timeout: aws.Int64(int64(timeout)),
Role: aws.String(role),
Runtime: aws.String(req.ResourceProperties["Runtime"].(string)),
Code: &lambda.FunctionCode{
ZipFile: body,
},
})
if err != nil {
return "", err
}
return *res.FunctionArn, nil
}
func S3Put(bucket, key string, data []byte, public bool) error {
req := &s3.PutObjectInput{
Body: bytes.NewReader(data),
Bucket: aws.String(bucket),
ContentLength: aws.Int64(int64(len(data))),
Key: aws.String(key),
}
if public {
req.ACL = aws.String("public-read")
}
_, err := S3().PutObject(req)
return err
}
func ECSServiceUpdate(req Request) (string, map[string]string, error) {
count, _ := strconv.Atoi(req.ResourceProperties["DesiredCount"].(string))
// arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY
parts := strings.Split(req.PhysicalResourceId, "/")
name := parts[1]
res, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{
Cluster: aws.String(req.ResourceProperties["Cluster"].(string)),
Service: aws.String(name),
DesiredCount: aws.Int64(int64(count)),
TaskDefinition: aws.String(req.ResourceProperties["TaskDefinition"].(string)),
})
if err != nil {
return "", nil, err
}
return *res.Service.ServiceArn, nil, nil
}
func ECSServiceDelete(req Request) (string, map[string]string, error) {
cluster := req.ResourceProperties["Cluster"].(string)
// arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY
parts := strings.Split(req.PhysicalResourceId, "/")
name := parts[1]
_, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{
Cluster: aws.String(cluster),
Service: aws.String(name),
DesiredCount: aws.Int64(0),
})
// go ahead and mark the delete good if the service is not found
if ae, ok := err.(awserr.Error); ok {
if ae.Code() == "ServiceNotFoundException" {
return req.PhysicalResourceId, nil, nil
}
}
// TODO let the cloudformation finish thinking this deleted
// but take note so we can figure out why
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return req.PhysicalResourceId, nil, nil
}
_, err = ECS(req).DeleteService(&ecs.DeleteServiceInput{
Cluster: aws.String(cluster),
Service: aws.String(name),
})
// TODO let the cloudformation finish thinking this deleted
// but take note so we can figure out why
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return req.PhysicalResourceId, nil, nil
}
return req.PhysicalResourceId, nil, nil
}
func getMetric(metric string, dimensions []*cloudwatch.Dimension, span, precision int64) ([]*cloudwatch.Datapoint, error) {
req := &cloudwatch.GetMetricStatisticsInput{
Dimensions: dimensions,
EndTime: aws.Time(time.Now()),
MetricName: aws.String(metric),
Namespace: aws.String("Convox"),
Period: aws.Int64(precision * 60),
StartTime: aws.Time(time.Now().Add(time.Duration(-1*span) * time.Minute)),
Statistics: []*string{aws.String("Average")},
}
res, err := CloudWatch().GetMetricStatistics(req)
if err != nil {
// TODO log error
fmt.Printf("error fetching metrics: %s\n", err)
return []*cloudwatch.Datapoint{}, nil
}
return res.Datapoints, nil
}
func S3PutFile(bucket, key string, f io.ReadSeeker, public bool) error {
// seek to end of f to determine length, then seek back to beginning for upload
l, err := f.Seek(0, 2)
if err != nil {
return err
}
_, err = f.Seek(0, 0)
if err != nil {
return err
}
req := &s3.PutObjectInput{
Body: f,
Bucket: aws.String(bucket),
ContentLength: aws.Int64(l),
Key: aws.String(key),
}
if public {
req.ACL = aws.String("public-read")
}
_, err = S3().PutObject(req)
if err != nil {
return err
}
// seek back to beginning in case something else needs to read f
_, err = f.Seek(0, 0)
return err
}
func ECSTaskDefinitionCreate(req Request) (string, map[string]string, error) {
// return "", fmt.Errorf("fail")
tasks := req.ResourceProperties["Tasks"].([]interface{})
r := &ecs.RegisterTaskDefinitionInput{
Family: aws.String(req.ResourceProperties["Name"].(string)),
}
// get environment from S3 URL
// 'Environment' is a CloudFormation Template Property that references 'Environment' CF Parameter with S3 URL
// S3 body may be encrypted with KMS key
var env models.Environment
if envUrl, ok := req.ResourceProperties["Environment"].(string); ok && envUrl != "" {
res, err := http.Get(envUrl)
if err != nil {
return "", nil, err
}
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if key, ok := req.ResourceProperties["Key"].(string); ok && key != "" {
cr := crypt.New(*Region(&req), os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"))
cr.AwsToken = os.Getenv("AWS_SESSION_TOKEN")
dec, err := cr.Decrypt(key, data)
if err != nil {
return "", nil, err
}
data = dec
}
env = models.LoadEnvironment(data)
}
r.ContainerDefinitions = make([]*ecs.ContainerDefinition, len(tasks))
for i, itask := range tasks {
task := itask.(map[string]interface{})
memory, _ := strconv.Atoi(task["Memory"].(string))
r.ContainerDefinitions[i] = &ecs.ContainerDefinition{
Name: aws.String(task["Name"].(string)),
Essential: aws.Bool(true),
Image: aws.String(task["Image"].(string)),
Memory: aws.Int64(int64(memory)),
}
if command, ok := task["Command"].(string); ok && command != "" {
r.ContainerDefinitions[i].Command = []*string{aws.String("sh"), aws.String("-c"), aws.String(command)}
}
// set Task environment from CF Tasks[].Environment key/values
// These key/values are read from the app manifest environment hash
if oenv, ok := task["Environment"].(map[string]interface{}); ok {
for key, val := range oenv {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String(key),
Value: aws.String(val.(string)),
})
}
}
// set Task environment from decrypted S3 URL body of key/values
// These key/values take precident over the above environment
for key, val := range env {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String(key),
Value: aws.String(val),
})
}
// set Release value in Task environment
if release, ok := req.ResourceProperties["Release"].(string); ok {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String("RELEASE"),
Value: aws.String(release),
})
}
// set links
if links, ok := task["Links"].([]interface{}); ok {
r.ContainerDefinitions[i].Links = make([]*string, len(links))
for j, link := range links {
r.ContainerDefinitions[i].Links[j] = aws.String(link.(string))
}
}
// set portmappings
if ports, ok := task["PortMappings"].([]interface{}); ok {
r.ContainerDefinitions[i].PortMappings = make([]*ecs.PortMapping, len(ports))
for j, port := range ports {
parts := strings.Split(port.(string), ":")
host, _ := strconv.Atoi(parts[0])
container, _ := strconv.Atoi(parts[1])
r.ContainerDefinitions[i].PortMappings[j] = &ecs.PortMapping{
ContainerPort: aws.Int64(int64(container)),
HostPort: aws.Int64(int64(host)),
}
}
}
// set volumes
if volumes, ok := task["Volumes"].([]interface{}); ok {
for j, volume := range volumes {
name := fmt.Sprintf("%s-%d-%d", task["Name"].(string), i, j)
parts := strings.Split(volume.(string), ":")
r.Volumes = append(r.Volumes, &ecs.Volume{
Name: aws.String(name),
Host: &ecs.HostVolumeProperties{
SourcePath: aws.String(parts[0]),
},
})
r.ContainerDefinitions[i].MountPoints = append(r.ContainerDefinitions[i].MountPoints, &ecs.MountPoint{
SourceVolume: aws.String(name),
ContainerPath: aws.String(parts[1]),
ReadOnly: aws.Bool(false),
})
}
}
}
res, err := ECS(req).RegisterTaskDefinition(r)
if err != nil {
return "", nil, err
}
return *res.TaskDefinition.TaskDefinitionArn, nil, nil
}
func subscribeRDS(prefix, id string, output chan []byte, quit chan bool) {
// Get latest log file details via pagination tokens
details := rds.DescribeDBLogFilesDetails{}
marker := ""
for {
params := &rds.DescribeDBLogFilesInput{
DBInstanceIdentifier: aws.String(id),
MaxRecords: aws.Int64(100),
}
if marker != "" {
params.Marker = aws.String(marker)
}
res, err := RDS().DescribeDBLogFiles(params)
if err != nil {
panic(err)
}
if res.Marker == nil {
files := res.DescribeDBLogFiles
details = *files[len(files)-1]
break
}
marker = *res.Marker
}
// Get last 50 log lines
params := &rds.DownloadDBLogFilePortionInput{
DBInstanceIdentifier: aws.String(id),
LogFileName: aws.String(*details.LogFileName),
NumberOfLines: aws.Int64(50),
}
res, err := RDS().DownloadDBLogFilePortion(params)
if err != nil {
panic(err)
}
output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))
params.Marker = aws.String(*res.Marker)
for {
select {
case <-quit:
fmt.Println("quitting")
return
default:
res, err := RDS().DownloadDBLogFilePortion(params)
if err != nil {
panic(err)
}
if *params.Marker != *res.Marker {
params.Marker = aws.String(*res.Marker)
output <- []byte(fmt.Sprintf("%s: %s\n", prefix, *res.LogFileData))
}
time.Sleep(1000 * time.Millisecond)
}
}
}