Example #1
0
func (a *App) UpdateParamsAndTemplate(changes map[string]string, template string) error {
	req := &cloudformation.UpdateStackInput{
		StackName:    aws.String(a.Name),
		Capabilities: []*string{aws.String("CAPABILITY_IAM")},
	}

	if template != "" {
		req.TemplateURL = aws.String(template)
	} else {
		req.UsePreviousTemplate = aws.Bool(true)
	}

	params := a.Parameters

	for key, val := range changes {
		params[key] = val
	}

	for key, val := range params {
		req.Parameters = append(req.Parameters, &cloudformation.Parameter{
			ParameterKey:   aws.String(key),
			ParameterValue: aws.String(val),
		})
	}

	_, err := CloudFormation().UpdateStack(req)

	return err
}
Example #2
0
func ListReleases(app string) (Releases, error) {
	req := &dynamodb.QueryInput{
		KeyConditions: map[string]*dynamodb.Condition{
			"app": &dynamodb.Condition{
				AttributeValueList: []*dynamodb.AttributeValue{
					&dynamodb.AttributeValue{S: aws.String(app)},
				},
				ComparisonOperator: aws.String("EQ"),
			},
		},
		IndexName:        aws.String("app.created"),
		Limit:            aws.Int64(20),
		ScanIndexForward: aws.Bool(false),
		TableName:        aws.String(releasesTable(app)),
	}

	res, err := DynamoDB().Query(req)

	if err != nil {
		return nil, err
	}

	releases := make(Releases, len(res.Items))

	for i, item := range res.Items {
		releases[i] = *releaseFromItem(item)
	}

	return releases, nil
}
Example #3
0
func (s *Service) UpdatePapertrail(arns map[string]string) error {
	input := struct {
		ARNs map[string]string
	}{
		arns,
	}

	formation, err := buildTemplate(fmt.Sprintf("service/%s", s.Type), "service", input)

	if err != nil {
		return err
	}

	// Update stack with all linked ARNs and EventSourceMappings
	_, err = CloudFormation().UpdateStack(&cloudformation.UpdateStackInput{
		StackName:    aws.String(s.Name),
		Capabilities: []*string{aws.String("CAPABILITY_IAM")},
		Parameters: []*cloudformation.Parameter{
			&cloudformation.Parameter{
				ParameterKey:   aws.String("Url"),
				ParameterValue: aws.String(s.Parameters["Url"]),
			},
		},
		TemplateBody: aws.String(formation),
	})

	return err
}
Example #4
0
func (b *Build) log(line string) {
	b.Logs += fmt.Sprintf("%s\n", line)

	if b.kinesis == "" {
		app, err := GetApp(b.App)

		if err != nil {
			panic(err)
		}

		b.kinesis = app.Outputs["Kinesis"]
	}

	_, err := Kinesis().PutRecords(&kinesis.PutRecordsInput{
		StreamName: aws.String(b.kinesis),
		Records: []*kinesis.PutRecordsRequestEntry{
			&kinesis.PutRecordsRequestEntry{
				Data:         []byte(fmt.Sprintf("build: %s", line)),
				PartitionKey: aws.String(string(time.Now().UnixNano())),
			},
		},
	})

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
	}
}
// Retrieve generates a new set of temporary credentials using STS.
func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {

	// Apply defaults where parameters are not set.
	if p.Client == nil {
		p.Client = sts.New(nil)
	}
	if p.RoleSessionName == "" {
		// Try to work out a role name that will hopefully end up unique.
		p.RoleSessionName = fmt.Sprintf("%d", time.Now().UTC().UnixNano())
	}
	if p.Duration == 0 {
		// Expire as often as AWS permits.
		p.Duration = 15 * time.Minute
	}

	roleOutput, err := p.Client.AssumeRole(&sts.AssumeRoleInput{
		DurationSeconds: aws.Int64(int64(p.Duration / time.Second)),
		RoleArn:         aws.String(p.RoleARN),
		RoleSessionName: aws.String(p.RoleSessionName),
		ExternalId:      p.ExternalID,
	})

	if err != nil {
		return credentials.Value{}, err
	}

	// We will proactively generate new credentials before they expire.
	p.SetExpiration(*roleOutput.Credentials.Expiration, p.ExpiryWindow)

	return credentials.Value{
		AccessKeyID:     *roleOutput.Credentials.AccessKeyId,
		SecretAccessKey: *roleOutput.Credentials.SecretAccessKey,
		SessionToken:    *roleOutput.Credentials.SessionToken,
	}, nil
}
Example #6
0
func (r *Release) Promote() error {
	formation, err := r.Formation()

	if err != nil {
		return err
	}

	existing, err := formationParameters(formation)

	if err != nil {
		return err
	}

	app, err := GetApp(r.App)

	if err != nil {
		return err
	}

	manifest, err := LoadManifest(r.Manifest)

	if err != nil {
		return err
	}

	for _, me := range manifest {
		app.Parameters[fmt.Sprintf("%sCommand", UpperName(me.Name))] = me.CommandString()
		app.Parameters[fmt.Sprintf("%sImage", UpperName(me.Name))] = fmt.Sprintf("%s/%s-%s:%s", os.Getenv("REGISTRY_HOST"), r.App, me.Name, r.Build)
	}

	app.Parameters["Environment"] = r.EnvironmentUrl()
	app.Parameters["Kernel"] = CustomTopic
	app.Parameters["Release"] = r.Id
	app.Parameters["Version"] = os.Getenv("RELEASE")

	if os.Getenv("ENCRYPTION_KEY") != "" {
		app.Parameters["Key"] = os.Getenv("ENCRYPTION_KEY")
	}

	params := []*cloudformation.Parameter{}

	for key, value := range app.Parameters {
		if _, ok := existing[key]; ok {
			params = append(params, &cloudformation.Parameter{ParameterKey: aws.String(key), ParameterValue: aws.String(value)})
		}
	}

	req := &cloudformation.UpdateStackInput{
		Capabilities: []*string{aws.String("CAPABILITY_IAM")},
		StackName:    aws.String(r.App),
		TemplateBody: aws.String(formation),
		Parameters:   params,
	}

	_, err = CloudFormation().UpdateStack(req)

	return err
}
Example #7
0
func Region(req *Request) *string {
	if req != nil {
		if region, ok := req.ResourceProperties["Region"].(string); ok && region != "" {
			return aws.String(region)
		}
	}

	return aws.String(os.Getenv("AWS_REGION"))
}
Example #8
0
func s3Delete(bucket, key string) error {
	req := &s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}

	_, err := S3().DeleteObject(req)

	return err
}
Example #9
0
func ListDeployments(app string) (Deployments, error) {
	a, err := GetApp(app)

	if err != nil {
		return nil, err
	}

	res, err := ECS().DescribeServices(&ecs.DescribeServicesInput{
		Cluster:  aws.String(os.Getenv("CLUSTER")),
		Services: []*string{aws.String(a.TaskDefinitionFamily())},
	})

	if err != nil {
		return nil, err
	}

	// no service yet, so no deployments
	if len(res.Services) != 1 {
		return Deployments{}, nil
	}

	service := res.Services[0]

	deployments := make(Deployments, len(service.Deployments))

	for i, d := range service.Deployments {
		deployments[i] = Deployment{
			Status:  *d.Status,
			Desired: *d.DesiredCount,
			Pending: *d.PendingCount,
			Running: *d.RunningCount,
			Created: *d.CreatedAt,
		}

		tres, err := ECS().DescribeTaskDefinition(&ecs.DescribeTaskDefinitionInput{
			TaskDefinition: d.TaskDefinition,
		})

		if err != nil {
			return nil, err
		}

		if len(tres.TaskDefinition.ContainerDefinitions) > 0 {
			for _, kp := range tres.TaskDefinition.ContainerDefinitions[0].Environment {
				if *kp.Name == "RELEASE" {
					deployments[i].Release = *kp.Value
					break
				}
			}
		}
	}

	return deployments, nil
}
Example #10
0
func KMSKeyCreate(req Request) (string, map[string]string, error) {
	res, err := KMS(req).CreateKey(&kms.CreateKeyInput{
		Description: aws.String(req.ResourceProperties["Description"].(string)),
		KeyUsage:    aws.String(req.ResourceProperties["KeyUsage"].(string)),
	})

	if err != nil {
		return "", nil, err
	}

	return *res.KeyMetadata.Arn, nil, nil
}
Example #11
0
func cleanupBucketObject(bucket, key, version string, S3 *s3.S3) {
	req := &s3.DeleteObjectInput{
		Bucket:    aws.String(bucket),
		Key:       aws.String(key),
		VersionId: aws.String(version),
	}

	_, err := S3.DeleteObject(req)

	if err != nil {
		fmt.Printf("error: %s\n", err)
	}
}
Example #12
0
func (p *Process) Stop() error {
	req := &ecs.StopTaskInput{
		Cluster: aws.String(os.Getenv("CLUSTER")),
		Task:    aws.String(p.taskArn),
	}

	_, err := ECS().StopTask(req)

	if err != nil {
		return err
	}

	return nil
}
Example #13
0
func s3Get(bucket, key string) ([]byte, error) {
	req := &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	}

	res, err := S3().GetObject(req)

	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(res.Body)
}
Example #14
0
func dequeueMessage() ([]Message, error) {
	req := &sqs.ReceiveMessageInput{
		MaxNumberOfMessages: aws.Int64(10),
		QueueUrl:            aws.String(MessageQueueUrl),
		WaitTimeSeconds:     aws.Int64(10),
	}

	res, err := SQS().ReceiveMessage(req)

	if err != nil {
		return nil, err
	}

	messages := make([]Message, len(res.Messages))

	var message Message

	for i, m := range res.Messages {
		err = json.Unmarshal([]byte(*m.Body), &message)

		if err != nil {
			return nil, err
		}

		message.MessageID = m.MessageId
		message.ReceiptHandle = m.ReceiptHandle

		messages[i] = message
	}

	return messages, nil
}
Example #15
0
func (s *Service) LinkPapertrail(app App) error {
	// build map of app name -> arn of all linked services
	arns := map[string]string{}

	for k, v := range s.Outputs {
		if strings.HasSuffix(k, "Link") {
			n := DashName(k)
			arns[n[:len(n)-5]] = v
		}
	}

	// get full Kinesis ARN for app
	req, err := Kinesis().DescribeStream(&kinesis.DescribeStreamInput{
		StreamName: aws.String(app.Outputs["Kinesis"]),
	})

	arn := *req.StreamDescription.StreamARN

	if err != nil {
		return err
	}

	// append new ARN and update
	arns[app.Name] = arn
	return s.UpdatePapertrail(arns)
}
Example #16
0
func subscribeKinesis(stream string, output chan []byte, quit chan bool) {
	sreq := &kinesis.DescribeStreamInput{
		StreamName: aws.String(stream),
	}
	sres, err := Kinesis().DescribeStream(sreq)

	if err != nil {
		fmt.Printf("err1 %+v\n", err)
		// panic(err)
		return
	}

	shards := make([]string, len(sres.StreamDescription.Shards))

	for i, s := range sres.StreamDescription.Shards {
		shards[i] = *s.ShardId
	}

	done := make([](chan bool), len(shards))

	for i, shard := range shards {
		done[i] = make(chan bool)
		go subscribeKinesisShard(stream, shard, output, done[i])
	}
}
Example #17
0
func S3Put(bucket, key string, data []byte, public bool) error {
	req := &s3.PutObjectInput{
		Body:          bytes.NewReader(data),
		Bucket:        aws.String(bucket),
		ContentLength: aws.Int64(int64(len(data))),
		Key:           aws.String(key),
	}

	if public {
		req.ACL = aws.String("public-read")
	}

	_, err := S3().PutObject(req)

	return err
}
Example #18
0
func subscribeKinesisShard(stream, shard string, output chan []byte, quit chan bool) {
	ireq := &kinesis.GetShardIteratorInput{
		ShardId:           aws.String(shard),
		ShardIteratorType: aws.String("LATEST"),
		StreamName:        aws.String(stream),
	}

	ires, err := Kinesis().GetShardIterator(ireq)

	if err != nil {
		fmt.Printf("err2 %+v\n", err)
		// panic(err)
		return
	}

	iter := *ires.ShardIterator

	for {
		select {
		case <-quit:
			fmt.Println("quitting")
			return
		default:
			greq := &kinesis.GetRecordsInput{
				ShardIterator: aws.String(iter),
			}
			gres, err := Kinesis().GetRecords(greq)

			if err != nil {
				fmt.Printf("err3 %+v\n", err)
				// panic(err)
				return
			}

			iter = *gres.NextShardIterator

			for _, record := range gres.Records {
				output <- []byte(fmt.Sprintf("%s\n", string(record.Data)))
			}

			time.Sleep(500 * time.Millisecond)
		}
	}
}
Example #19
0
func DockerHost() (string, error) {
	ares, err := ECS().ListContainerInstances(&ecs.ListContainerInstancesInput{
		Cluster: aws.String(os.Getenv("CLUSTER")),
	})

	if len(ares.ContainerInstanceArns) == 0 {
		return "", fmt.Errorf("no container instances")
	}

	cres, err := ECS().DescribeContainerInstances(&ecs.DescribeContainerInstancesInput{
		Cluster:            aws.String(os.Getenv("CLUSTER")),
		ContainerInstances: ares.ContainerInstanceArns,
	})

	if err != nil {
		return "", err
	}

	if len(cres.ContainerInstances) == 0 {
		return "", fmt.Errorf("no container instances")
	}

	id := *cres.ContainerInstances[rand.Intn(len(cres.ContainerInstances))].Ec2InstanceId

	ires, err := EC2().DescribeInstances(&ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{Name: aws.String("instance-id"), Values: []*string{&id}},
		},
	})

	if len(ires.Reservations) != 1 || len(ires.Reservations[0].Instances) != 1 {
		return "", fmt.Errorf("could not describe container instance")
	}

	ip := *ires.Reservations[0].Instances[0].PrivateIpAddress

	if os.Getenv("DEVELOPMENT") == "true" {
		ip = *ires.Reservations[0].Instances[0].PublicIpAddress
	}

	return fmt.Sprintf("http://%s:2376", ip), nil
}
Example #20
0
func LambdaFunctionCreate(req Request) (string, error) {
	bres, err := http.Get(req.ResourceProperties["ZipFile"].(string))

	if err != nil {
		return "", err
	}

	defer bres.Body.Close()

	body, err := ioutil.ReadAll(bres.Body)

	memory := 128
	timeout := 5

	if m, ok := req.ResourceProperties["Memory"].(string); ok && m != "" {
		memory, _ = strconv.Atoi(m)
	}

	if t, ok := req.ResourceProperties["Timeout"].(string); ok && t != "" {
		timeout, _ = strconv.Atoi(t)
	}

	role := fmt.Sprintf("arn:aws:iam::%s:role/%s", req.ResourceProperties["AccountId"].(string), req.ResourceProperties["Role"].(string))

	res, err := Lambda(req).CreateFunction(&lambda.CreateFunctionInput{
		FunctionName: aws.String(req.ResourceProperties["Name"].(string)),
		Handler:      aws.String(req.ResourceProperties["Handler"].(string)),
		MemorySize:   aws.Int64(int64(memory)),
		Timeout:      aws.Int64(int64(timeout)),
		Role:         aws.String(role),
		Runtime:      aws.String(req.ResourceProperties["Runtime"].(string)),
		Code: &lambda.FunctionCode{
			ZipFile: body,
		},
	})

	if err != nil {
		return "", err
	}

	return *res.FunctionArn, nil
}
Example #21
0
func ECSServiceUpdate(req Request) (string, map[string]string, error) {
	count, _ := strconv.Atoi(req.ResourceProperties["DesiredCount"].(string))

	// arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY
	parts := strings.Split(req.PhysicalResourceId, "/")
	name := parts[1]

	res, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{
		Cluster:        aws.String(req.ResourceProperties["Cluster"].(string)),
		Service:        aws.String(name),
		DesiredCount:   aws.Int64(int64(count)),
		TaskDefinition: aws.String(req.ResourceProperties["TaskDefinition"].(string)),
	})

	if err != nil {
		return "", nil, err
	}

	return *res.Service.ServiceArn, nil, nil
}
Example #22
0
func (s *Service) Delete() error {
	name := s.Name

	_, err := CloudFormation().DeleteStack(&cloudformation.DeleteStackInput{StackName: aws.String(name)})

	if err != nil {
		return err
	}

	return nil
}
Example #23
0
func ECSClusterCreate(req Request) (string, map[string]string, error) {
	res, err := ECS(req).CreateCluster(&ecs.CreateClusterInput{
		ClusterName: aws.String(req.ResourceProperties["Name"].(string)),
	})

	if err != nil {
		return "", nil, err
	}

	return *res.Cluster.ClusterArn, nil, nil
}
Example #24
0
func ECSServiceDelete(req Request) (string, map[string]string, error) {
	cluster := req.ResourceProperties["Cluster"].(string)

	// arn:aws:ecs:us-east-1:922560784203:service/sinatra-SZXTRXEMYEY
	parts := strings.Split(req.PhysicalResourceId, "/")
	name := parts[1]

	_, err := ECS(req).UpdateService(&ecs.UpdateServiceInput{
		Cluster:      aws.String(cluster),
		Service:      aws.String(name),
		DesiredCount: aws.Int64(0),
	})

	// go ahead and mark the delete good if the service is not found
	if ae, ok := err.(awserr.Error); ok {
		if ae.Code() == "ServiceNotFoundException" {
			return req.PhysicalResourceId, nil, nil
		}
	}

	// TODO let the cloudformation finish thinking this deleted
	// but take note so we can figure out why
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return req.PhysicalResourceId, nil, nil
	}

	_, err = ECS(req).DeleteService(&ecs.DeleteServiceInput{
		Cluster: aws.String(cluster),
		Service: aws.String(name),
	})

	// TODO let the cloudformation finish thinking this deleted
	// but take note so we can figure out why
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return req.PhysicalResourceId, nil, nil
	}

	return req.PhysicalResourceId, nil, nil
}
Example #25
0
func (r *Release) Save() error {
	if r.Id == "" {
		return fmt.Errorf("Id must not be blank")
	}

	if r.Created.IsZero() {
		r.Created = time.Now()
	}

	req := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			"id":      &dynamodb.AttributeValue{S: aws.String(r.Id)},
			"app":     &dynamodb.AttributeValue{S: aws.String(r.App)},
			"created": &dynamodb.AttributeValue{S: aws.String(r.Created.Format(SortableTime))},
		},
		TableName: aws.String(releasesTable(r.App)),
	}

	if r.Build != "" {
		req.Item["build"] = &dynamodb.AttributeValue{S: aws.String(r.Build)}
	}

	if r.Env != "" {
		req.Item["env"] = &dynamodb.AttributeValue{S: aws.String(r.Env)}
	}

	if r.Manifest != "" {
		req.Item["manifest"] = &dynamodb.AttributeValue{S: aws.String(r.Manifest)}
	}

	_, err := DynamoDB().PutItem(req)

	if err != nil {
		return err
	}

	app, err := GetApp(r.App)

	if err != nil {
		return err
	}

	env := []byte(r.Env)

	if app.Parameters["Key"] != "" {
		cr := crypt.New(os.Getenv("AWS_REGION"), os.Getenv("AWS_ACCESS"), os.Getenv("AWS_SECRET"))

		env, err = cr.Encrypt(app.Parameters["Key"], []byte(env))

		if err != nil {
			return err
		}
	}

	return S3Put(app.Outputs["Settings"], fmt.Sprintf("releases/%s/env", r.Id), env, true)
}
Example #26
0
func getMetric(metric string, dimensions []*cloudwatch.Dimension, span, precision int64) ([]*cloudwatch.Datapoint, error) {
	req := &cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		EndTime:    aws.Time(time.Now()),
		MetricName: aws.String(metric),
		Namespace:  aws.String("Convox"),
		Period:     aws.Int64(precision * 60),
		StartTime:  aws.Time(time.Now().Add(time.Duration(-1*span) * time.Minute)),
		Statistics: []*string{aws.String("Average")},
	}

	res, err := CloudWatch().GetMetricStatistics(req)

	if err != nil {
		// TODO log error
		fmt.Printf("error fetching metrics: %s\n", err)
		return []*cloudwatch.Datapoint{}, nil
	}

	return res.Datapoints, nil
}
Example #27
0
func (a *App) RunDetached(process, command string) error {
	resources := a.Resources()

	req := &ecs.RunTaskInput{
		Cluster:        aws.String(os.Getenv("CLUSTER")),
		Count:          aws.Int64(1),
		TaskDefinition: aws.String(resources[UpperName(process)+"ECSTaskDefinition"].Id),
	}

	if command != "" {
		req.Overrides = &ecs.TaskOverride{
			ContainerOverrides: []*ecs.ContainerOverride{
				&ecs.ContainerOverride{
					Name: aws.String(process),
					Command: []*string{
						aws.String("sh"),
						aws.String("-c"),
						aws.String(command),
					},
				},
			},
		}
	}

	_, err := ECS().RunTask(req)

	if err != nil {
		return err
	}

	return nil
}
Example #28
0
func InstanceMetrics(app, process, instance string) (*Metrics, error) {
	dimensions := []*cloudwatch.Dimension{
		&cloudwatch.Dimension{Name: aws.String("App"), Value: aws.String(app)},
		&cloudwatch.Dimension{Name: aws.String("Process"), Value: aws.String(process)},
		&cloudwatch.Dimension{Name: aws.String("InstanceId"), Value: aws.String(instance)},
	}

	cpu, err := getMetric("CpuUtilization", dimensions, 1, 1)

	if err != nil {
		return nil, err
	}

	memory, err := getMetric("MemoryUtilization", dimensions, 1, 1)

	if err != nil {
		return nil, err
	}

	disk, err := getMetric("DiskUtilization", dimensions, 1, 1)

	if err != nil {
		return nil, err
	}

	return &Metrics{Cpu: getLastAverage(cpu), Memory: getLastAverage(memory), Disk: getLastAverage(disk)}, nil
}
Example #29
0
func EC2AvailabilityZonesCreate(req Request) (string, map[string]string, error) {
	_, err := EC2(req).CreateSubnet(&ec2.CreateSubnetInput{
		AvailabilityZone: aws.String("garbage"),
		CidrBlock:        aws.String("10.200.0.0/16"),
		VpcId:            aws.String(req.ResourceProperties["Vpc"].(string)),
	})

	matches := regexMatchAvailabilityZones.FindStringSubmatch(err.Error())
	matches = strings.Split(strings.Replace(matches[1], " ", "", -1), ",")

	if len(matches) < 1 {
		return "", nil, fmt.Errorf("could not discover availability zones")
	}

	outputs := make(map[string]string)
	for i, az := range matches {
		outputs["AvailabilityZone"+strconv.Itoa(i)] = az
	}

	physical := strings.Join(matches, ",")
	return physical, outputs, nil
}
Example #30
0
func GetRelease(app, id string) (*Release, error) {
	req := &dynamodb.GetItemInput{
		ConsistentRead: aws.Bool(true),
		Key: map[string]*dynamodb.AttributeValue{
			"id": &dynamodb.AttributeValue{S: aws.String(id)},
		},
		TableName: aws.String(releasesTable(app)),
	}

	res, err := DynamoDB().GetItem(req)

	if err != nil {
		return nil, err
	}

	if res.Item == nil {
		return nil, fmt.Errorf("no such release: %s", id)
	}

	release := releaseFromItem(res.Item)

	return release, nil
}