func (a *App) UpdateParamsAndTemplate(changes map[string]string, template string) error {
req := &cloudformation.UpdateStackInput{
StackName: aws.String(a.Name),
Capabilities: []*string{aws.String("CAPABILITY_IAM")},
}
if template != "" {
req.TemplateURL = aws.String(template)
} else {
req.UsePreviousTemplate = aws.Bool(true)
}
params := a.Parameters
for key, val := range changes {
params[key] = val
}
for key, val := range params {
req.Parameters = append(req.Parameters, &cloudformation.Parameter{
ParameterKey: aws.String(key),
ParameterValue: aws.String(val),
})
}
_, err := CloudFormation().UpdateStack(req)
return err
}
func ListReleases(app string) (Releases, error) {
req := &dynamodb.QueryInput{
KeyConditions: map[string]*dynamodb.Condition{
"app": &dynamodb.Condition{
AttributeValueList: []*dynamodb.AttributeValue{
&dynamodb.AttributeValue{S: aws.String(app)},
},
ComparisonOperator: aws.String("EQ"),
},
},
IndexName: aws.String("app.created"),
Limit: aws.Int64(20),
ScanIndexForward: aws.Bool(false),
TableName: aws.String(releasesTable(app)),
}
res, err := DynamoDB().Query(req)
if err != nil {
return nil, err
}
releases := make(Releases, len(res.Items))
for i, item := range res.Items {
releases[i] = *releaseFromItem(item)
}
return releases, nil
}
func validateCRC32(r *request.Request) {
if r.Error != nil {
return // already have an error, no need to verify CRC
}
// Checksum validation is off, skip
if aws.BoolValue(r.Service.Config.DisableComputeChecksums) {
return
}
// Try to get CRC from response
header := r.HTTPResponse.Header.Get("X-Amz-Crc32")
if header == "" {
return // No header, skip
}
expected, err := strconv.ParseUint(header, 10, 32)
if err != nil {
return // Could not determine CRC value, skip
}
buf, err := drainBody(r.HTTPResponse.Body)
if err != nil { // failed to read the response body, skip
return
}
// Reset body for subsequent reads
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader(buf.Bytes()))
// Compute the CRC checksum
crc := crc32.ChecksumIEEE(buf.Bytes())
if crc != uint32(expected) {
// CRC does not match, set a retryable error
r.Retryable = aws.Bool(true)
r.Error = awserr.New("CRC32CheckFailed", "CRC32 integrity check failed", nil)
}
}
func GetRelease(app, id string) (*Release, error) {
req := &dynamodb.GetItemInput{
ConsistentRead: aws.Bool(true),
Key: map[string]*dynamodb.AttributeValue{
"id": &dynamodb.AttributeValue{S: aws.String(id)},
},
TableName: aws.String(releasesTable(app)),
}
res, err := DynamoDB().GetItem(req)
if err != nil {
return nil, err
}
if res.Item == nil {
return nil, fmt.Errorf("no such release: %s", id)
}
release := releaseFromItem(res.Item)
return release, nil
}
func GetBuild(app, id string) (*Build, error) {
req := &dynamodb.GetItemInput{
ConsistentRead: aws.Bool(true),
Key: map[string]*dynamodb.AttributeValue{
"id": &dynamodb.AttributeValue{S: aws.String(id)},
},
TableName: aws.String(buildsTable(app)),
}
res, err := DynamoDB().GetItem(req)
if err != nil {
return nil, err
}
if res.Item == nil {
return nil, fmt.Errorf("no such build: %s", id)
}
build := buildFromItem(res.Item)
return build, nil
}
func ECSTaskDefinitionCreate(req Request) (string, map[string]string, error) {
// return "", fmt.Errorf("fail")
tasks := req.ResourceProperties["Tasks"].([]interface{})
r := &ecs.RegisterTaskDefinitionInput{
Family: aws.String(req.ResourceProperties["Name"].(string)),
}
// get environment from S3 URL
// 'Environment' is a CloudFormation Template Property that references 'Environment' CF Parameter with S3 URL
// S3 body may be encrypted with KMS key
var env models.Environment
if envUrl, ok := req.ResourceProperties["Environment"].(string); ok && envUrl != "" {
res, err := http.Get(envUrl)
if err != nil {
return "", nil, err
}
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if key, ok := req.ResourceProperties["Key"].(string); ok && key != "" {
cr := crypt.New(*Region(&req), os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"))
cr.AwsToken = os.Getenv("AWS_SESSION_TOKEN")
dec, err := cr.Decrypt(key, data)
if err != nil {
return "", nil, err
}
data = dec
}
env = models.LoadEnvironment(data)
}
r.ContainerDefinitions = make([]*ecs.ContainerDefinition, len(tasks))
for i, itask := range tasks {
task := itask.(map[string]interface{})
memory, _ := strconv.Atoi(task["Memory"].(string))
r.ContainerDefinitions[i] = &ecs.ContainerDefinition{
Name: aws.String(task["Name"].(string)),
Essential: aws.Bool(true),
Image: aws.String(task["Image"].(string)),
Memory: aws.Int64(int64(memory)),
}
if command, ok := task["Command"].(string); ok && command != "" {
r.ContainerDefinitions[i].Command = []*string{aws.String("sh"), aws.String("-c"), aws.String(command)}
}
// set Task environment from CF Tasks[].Environment key/values
// These key/values are read from the app manifest environment hash
if oenv, ok := task["Environment"].(map[string]interface{}); ok {
for key, val := range oenv {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String(key),
Value: aws.String(val.(string)),
})
}
}
// set Task environment from decrypted S3 URL body of key/values
// These key/values take precident over the above environment
for key, val := range env {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String(key),
Value: aws.String(val),
})
}
// set Release value in Task environment
if release, ok := req.ResourceProperties["Release"].(string); ok {
r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{
Name: aws.String("RELEASE"),
Value: aws.String(release),
})
}
// set links
if links, ok := task["Links"].([]interface{}); ok {
r.ContainerDefinitions[i].Links = make([]*string, len(links))
for j, link := range links {
r.ContainerDefinitions[i].Links[j] = aws.String(link.(string))
}
}
// set portmappings
if ports, ok := task["PortMappings"].([]interface{}); ok {
r.ContainerDefinitions[i].PortMappings = make([]*ecs.PortMapping, len(ports))
for j, port := range ports {
parts := strings.Split(port.(string), ":")
host, _ := strconv.Atoi(parts[0])
container, _ := strconv.Atoi(parts[1])
r.ContainerDefinitions[i].PortMappings[j] = &ecs.PortMapping{
ContainerPort: aws.Int64(int64(container)),
HostPort: aws.Int64(int64(host)),
}
}
}
// set volumes
if volumes, ok := task["Volumes"].([]interface{}); ok {
for j, volume := range volumes {
name := fmt.Sprintf("%s-%d-%d", task["Name"].(string), i, j)
parts := strings.Split(volume.(string), ":")
r.Volumes = append(r.Volumes, &ecs.Volume{
Name: aws.String(name),
Host: &ecs.HostVolumeProperties{
SourcePath: aws.String(parts[0]),
},
})
r.ContainerDefinitions[i].MountPoints = append(r.ContainerDefinitions[i].MountPoints, &ecs.MountPoint{
SourceVolume: aws.String(name),
ContainerPath: aws.String(parts[1]),
ReadOnly: aws.Bool(false),
})
}
}
}
res, err := ECS(req).RegisterTaskDefinition(r)
if err != nil {
return "", nil, err
}
return *res.TaskDefinition.TaskDefinitionArn, nil, nil
}
func StartCluster() {
var log = logger.New("ns=cluster_monitor")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
Tick:
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
// Ger Rack InstanceCount Parameter
instanceCount := 0
// instanceType := "unknown"
res, err := models.CloudFormation().DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String(os.Getenv("RACK")),
},
)
if err != nil {
log.Error(err)
continue
}
for _, p := range res.Stacks[0].Parameters {
if *p.ParameterKey == "InstanceCount" {
c, err := strconv.Atoi(*p.ParameterValue)
if err != nil {
log.Error(err)
break Tick
}
instanceCount = c
}
// if *p.ParameterKey == "InstanceType" {
// instanceType = *p.ParameterValue
// }
}
// helpers.SendMixpanelEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", instanceCount, instanceType))
// List and Describe ECS Container Instances
ires, err := models.ECS().ListContainerInstances(
&ecs.ListContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
},
)
if err != nil {
log.Error(err)
continue
}
dres, err := models.ECS().DescribeContainerInstances(
&ecs.DescribeContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
ContainerInstances: ires.ContainerInstanceArns,
},
)
if err != nil {
log.Error(err)
continue
}
cInstanceIds := make([]string, 0)
cInstanceConnections := make(map[string]bool)
for _, i := range dres.ContainerInstances {
cInstanceConnections[*i.Ec2InstanceId] = *i.AgentConnected
if *i.AgentConnected {
cInstanceIds = append(cInstanceIds, *i.Ec2InstanceId)
}
}
// Get and Describe Rack ASG Resource
resources, err := models.ListResources(os.Getenv("RACK"))
ares, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
aInstanceIds := []string{}
uInstanceIds := []string{}
for _, i := range ares.AutoScalingGroups[0].Instances {
if connected, exists := cInstanceConnections[*i.InstanceId]; connected && exists {
aInstanceIds = append(aInstanceIds, *i.InstanceId)
} else {
// Not registered or not connected => set Unhealthy
if *i.LifecycleState == "InService" {
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceId: aws.String(*i.InstanceId),
ShouldRespectGracePeriod: aws.Bool(true),
},
)
if err != nil {
log.Error(err)
continue
}
uInstanceIds = append(uInstanceIds, *i.InstanceId)
}
}
}
sort.Strings(aInstanceIds)
sort.Strings(cInstanceIds)
sort.Strings(uInstanceIds)
// if len(uInstanceIds) > 0 {
// helpers.SendMixpanelEvent("kernel-cluster-monitor-mark", strings.Join(uInstanceIds, ","))
// }
log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
}
}
func setChecksumError(r *request.Request, format string, args ...interface{}) {
r.Retryable = aws.Bool(true)
r.Error = awserr.New("InvalidChecksum", fmt.Sprintf(format, args...), nil)
}
}
return
}
}
if r.HTTPResponse == nil {
// Add a dummy request response object to ensure the HTTPResponse
// value is consistent.
r.HTTPResponse = &http.Response{
StatusCode: int(0),
Status: http.StatusText(int(0)),
Body: ioutil.NopCloser(bytes.NewReader([]byte{})),
}
}
// Catch all other request errors.
r.Error = awserr.New("RequestError", "send request failed", err)
r.Retryable = aws.Bool(true) // network errors are retryable
}
}}
// ValidateResponseHandler is a request handler to validate service response.
var ValidateResponseHandler = request.NamedHandler{"core.ValidateResponseHandler", func(r *request.Request) {
if r.HTTPResponse.StatusCode == 0 || r.HTTPResponse.StatusCode >= 300 {
// this may be replaced by an UnmarshalError handler
r.Error = awserr.New("UnknownError", "unknown error", nil)
}
}}
// AfterRetryHandler performs final checks to determine if the request should
// be retried and how long to delay.
var AfterRetryHandler = request.NamedHandler{"core.AfterRetryHandler", func(r *request.Request) {
// If one of the other handlers already set the retry state