func (a *App) UpdateParamsAndTemplate(changes map[string]string, template string) error { req := &cloudformation.UpdateStackInput{ StackName: aws.String(a.Name), Capabilities: []*string{aws.String("CAPABILITY_IAM")}, } if template != "" { req.TemplateURL = aws.String(template) } else { req.UsePreviousTemplate = aws.Bool(true) } params := a.Parameters for key, val := range changes { params[key] = val } for key, val := range params { req.Parameters = append(req.Parameters, &cloudformation.Parameter{ ParameterKey: aws.String(key), ParameterValue: aws.String(val), }) } _, err := CloudFormation().UpdateStack(req) return err }
func ListReleases(app string) (Releases, error) { req := &dynamodb.QueryInput{ KeyConditions: map[string]*dynamodb.Condition{ "app": &dynamodb.Condition{ AttributeValueList: []*dynamodb.AttributeValue{ &dynamodb.AttributeValue{S: aws.String(app)}, }, ComparisonOperator: aws.String("EQ"), }, }, IndexName: aws.String("app.created"), Limit: aws.Int64(20), ScanIndexForward: aws.Bool(false), TableName: aws.String(releasesTable(app)), } res, err := DynamoDB().Query(req) if err != nil { return nil, err } releases := make(Releases, len(res.Items)) for i, item := range res.Items { releases[i] = *releaseFromItem(item) } return releases, nil }
func validateCRC32(r *request.Request) { if r.Error != nil { return // already have an error, no need to verify CRC } // Checksum validation is off, skip if aws.BoolValue(r.Service.Config.DisableComputeChecksums) { return } // Try to get CRC from response header := r.HTTPResponse.Header.Get("X-Amz-Crc32") if header == "" { return // No header, skip } expected, err := strconv.ParseUint(header, 10, 32) if err != nil { return // Could not determine CRC value, skip } buf, err := drainBody(r.HTTPResponse.Body) if err != nil { // failed to read the response body, skip return } // Reset body for subsequent reads r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader(buf.Bytes())) // Compute the CRC checksum crc := crc32.ChecksumIEEE(buf.Bytes()) if crc != uint32(expected) { // CRC does not match, set a retryable error r.Retryable = aws.Bool(true) r.Error = awserr.New("CRC32CheckFailed", "CRC32 integrity check failed", nil) } }
func GetRelease(app, id string) (*Release, error) { req := &dynamodb.GetItemInput{ ConsistentRead: aws.Bool(true), Key: map[string]*dynamodb.AttributeValue{ "id": &dynamodb.AttributeValue{S: aws.String(id)}, }, TableName: aws.String(releasesTable(app)), } res, err := DynamoDB().GetItem(req) if err != nil { return nil, err } if res.Item == nil { return nil, fmt.Errorf("no such release: %s", id) } release := releaseFromItem(res.Item) return release, nil }
func GetBuild(app, id string) (*Build, error) { req := &dynamodb.GetItemInput{ ConsistentRead: aws.Bool(true), Key: map[string]*dynamodb.AttributeValue{ "id": &dynamodb.AttributeValue{S: aws.String(id)}, }, TableName: aws.String(buildsTable(app)), } res, err := DynamoDB().GetItem(req) if err != nil { return nil, err } if res.Item == nil { return nil, fmt.Errorf("no such build: %s", id) } build := buildFromItem(res.Item) return build, nil }
func ECSTaskDefinitionCreate(req Request) (string, map[string]string, error) { // return "", fmt.Errorf("fail") tasks := req.ResourceProperties["Tasks"].([]interface{}) r := &ecs.RegisterTaskDefinitionInput{ Family: aws.String(req.ResourceProperties["Name"].(string)), } // get environment from S3 URL // 'Environment' is a CloudFormation Template Property that references 'Environment' CF Parameter with S3 URL // S3 body may be encrypted with KMS key var env models.Environment if envUrl, ok := req.ResourceProperties["Environment"].(string); ok && envUrl != "" { res, err := http.Get(envUrl) if err != nil { return "", nil, err } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if key, ok := req.ResourceProperties["Key"].(string); ok && key != "" { cr := crypt.New(*Region(&req), os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")) cr.AwsToken = os.Getenv("AWS_SESSION_TOKEN") dec, err := cr.Decrypt(key, data) if err != nil { return "", nil, err } data = dec } env = models.LoadEnvironment(data) } r.ContainerDefinitions = make([]*ecs.ContainerDefinition, len(tasks)) for i, itask := range tasks { task := itask.(map[string]interface{}) memory, _ := strconv.Atoi(task["Memory"].(string)) r.ContainerDefinitions[i] = &ecs.ContainerDefinition{ Name: aws.String(task["Name"].(string)), Essential: aws.Bool(true), Image: aws.String(task["Image"].(string)), Memory: aws.Int64(int64(memory)), } if command, ok := task["Command"].(string); ok && command != "" { r.ContainerDefinitions[i].Command = []*string{aws.String("sh"), aws.String("-c"), aws.String(command)} } // set Task environment from CF Tasks[].Environment key/values // These key/values are read from the app manifest environment hash if oenv, ok := task["Environment"].(map[string]interface{}); ok { for key, val := range oenv { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String(key), Value: aws.String(val.(string)), }) } } // set Task environment from decrypted S3 URL body of key/values // These key/values take precident over the above environment for key, val := range env { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String(key), Value: aws.String(val), }) } // set Release value in Task environment if release, ok := req.ResourceProperties["Release"].(string); ok { r.ContainerDefinitions[i].Environment = append(r.ContainerDefinitions[i].Environment, &ecs.KeyValuePair{ Name: aws.String("RELEASE"), Value: aws.String(release), }) } // set links if links, ok := task["Links"].([]interface{}); ok { r.ContainerDefinitions[i].Links = make([]*string, len(links)) for j, link := range links { r.ContainerDefinitions[i].Links[j] = aws.String(link.(string)) } } // set portmappings if ports, ok := task["PortMappings"].([]interface{}); ok { r.ContainerDefinitions[i].PortMappings = make([]*ecs.PortMapping, len(ports)) for j, port := range ports { parts := strings.Split(port.(string), ":") host, _ := strconv.Atoi(parts[0]) container, _ := strconv.Atoi(parts[1]) r.ContainerDefinitions[i].PortMappings[j] = &ecs.PortMapping{ ContainerPort: aws.Int64(int64(container)), HostPort: aws.Int64(int64(host)), } } } // set volumes if volumes, ok := task["Volumes"].([]interface{}); ok { for j, volume := range volumes { name := fmt.Sprintf("%s-%d-%d", task["Name"].(string), i, j) parts := strings.Split(volume.(string), ":") r.Volumes = append(r.Volumes, &ecs.Volume{ Name: aws.String(name), Host: &ecs.HostVolumeProperties{ SourcePath: aws.String(parts[0]), }, }) r.ContainerDefinitions[i].MountPoints = append(r.ContainerDefinitions[i].MountPoints, &ecs.MountPoint{ SourceVolume: aws.String(name), ContainerPath: aws.String(parts[1]), ReadOnly: aws.Bool(false), }) } } } res, err := ECS(req).RegisterTaskDefinition(r) if err != nil { return "", nil, err } return *res.TaskDefinition.TaskDefinitionArn, nil, nil }
func StartCluster() { var log = logger.New("ns=cluster_monitor") defer recoverWith(func(err error) { helpers.Error(log, err) }) Tick: for _ = range time.Tick(5 * time.Minute) { log.Log("tick") // Ger Rack InstanceCount Parameter instanceCount := 0 // instanceType := "unknown" res, err := models.CloudFormation().DescribeStacks( &cloudformation.DescribeStacksInput{ StackName: aws.String(os.Getenv("RACK")), }, ) if err != nil { log.Error(err) continue } for _, p := range res.Stacks[0].Parameters { if *p.ParameterKey == "InstanceCount" { c, err := strconv.Atoi(*p.ParameterValue) if err != nil { log.Error(err) break Tick } instanceCount = c } // if *p.ParameterKey == "InstanceType" { // instanceType = *p.ParameterValue // } } // helpers.SendMixpanelEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", instanceCount, instanceType)) // List and Describe ECS Container Instances ires, err := models.ECS().ListContainerInstances( &ecs.ListContainerInstancesInput{ Cluster: aws.String(os.Getenv("CLUSTER")), }, ) if err != nil { log.Error(err) continue } dres, err := models.ECS().DescribeContainerInstances( &ecs.DescribeContainerInstancesInput{ Cluster: aws.String(os.Getenv("CLUSTER")), ContainerInstances: ires.ContainerInstanceArns, }, ) if err != nil { log.Error(err) continue } cInstanceIds := make([]string, 0) cInstanceConnections := make(map[string]bool) for _, i := range dres.ContainerInstances { cInstanceConnections[*i.Ec2InstanceId] = *i.AgentConnected if *i.AgentConnected { cInstanceIds = append(cInstanceIds, *i.Ec2InstanceId) } } // Get and Describe Rack ASG Resource resources, err := models.ListResources(os.Getenv("RACK")) ares, err := models.AutoScaling().DescribeAutoScalingGroups( &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: []*string{ aws.String(resources["Instances"].Id), }, }, ) if err != nil { log.Error(err) continue } // Test if ASG Instance is registered and connected in ECS cluster aInstanceIds := []string{} uInstanceIds := []string{} for _, i := range ares.AutoScalingGroups[0].Instances { if connected, exists := cInstanceConnections[*i.InstanceId]; connected && exists { aInstanceIds = append(aInstanceIds, *i.InstanceId) } else { // Not registered or not connected => set Unhealthy if *i.LifecycleState == "InService" { _, err := models.AutoScaling().SetInstanceHealth( &autoscaling.SetInstanceHealthInput{ HealthStatus: aws.String("Unhealthy"), InstanceId: aws.String(*i.InstanceId), ShouldRespectGracePeriod: aws.Bool(true), }, ) if err != nil { log.Error(err) continue } uInstanceIds = append(uInstanceIds, *i.InstanceId) } } } sort.Strings(aInstanceIds) sort.Strings(cInstanceIds) sort.Strings(uInstanceIds) // if len(uInstanceIds) > 0 { // helpers.SendMixpanelEvent("kernel-cluster-monitor-mark", strings.Join(uInstanceIds, ",")) // } log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ",")) } }
func setChecksumError(r *request.Request, format string, args ...interface{}) { r.Retryable = aws.Bool(true) r.Error = awserr.New("InvalidChecksum", fmt.Sprintf(format, args...), nil) }
} return } } if r.HTTPResponse == nil { // Add a dummy request response object to ensure the HTTPResponse // value is consistent. r.HTTPResponse = &http.Response{ StatusCode: int(0), Status: http.StatusText(int(0)), Body: ioutil.NopCloser(bytes.NewReader([]byte{})), } } // Catch all other request errors. r.Error = awserr.New("RequestError", "send request failed", err) r.Retryable = aws.Bool(true) // network errors are retryable } }} // ValidateResponseHandler is a request handler to validate service response. var ValidateResponseHandler = request.NamedHandler{"core.ValidateResponseHandler", func(r *request.Request) { if r.HTTPResponse.StatusCode == 0 || r.HTTPResponse.StatusCode >= 300 { // this may be replaced by an UnmarshalError handler r.Error = awserr.New("UnknownError", "unknown error", nil) } }} // AfterRetryHandler performs final checks to determine if the request should // be retried and how long to delay. var AfterRetryHandler = request.NamedHandler{"core.AfterRetryHandler", func(r *request.Request) { // If one of the other handlers already set the retry state