Ejemplo n.º 1
0
func ClusterTop(rw http.ResponseWriter, r *http.Request) {
	name := aws.String(os.Getenv("RACK"))

	res, err := models.CloudFormation().DescribeStacks(&cloudformation.DescribeStacksInput{StackName: name})

	if err != nil {
		RenderError(rw, err)
		return
	}

	if len(res.Stacks) == 0 {
		RenderError(rw, fmt.Errorf("Stack %s does not exist", os.Getenv("RACK")))
		return
	}

	stack := res.Stacks[0]

	outputs := make(map[string]string)

	for _, output := range stack.Outputs {
		outputs[*output.OutputKey] = *output.OutputValue
	}

	cluster := outputs["Cluster"]

	params := &cloudwatch.GetMetricStatisticsInput{
		MetricName: aws.String(mux.Vars(r)["metric"]),
		StartTime:  aws.Time(time.Now().Add(-2 * time.Minute)),
		EndTime:    aws.Time(time.Now()),
		Period:     aws.Long(60),
		Namespace:  aws.String("AWS/ECS"),
		Statistics: []*string{ // Required
			aws.String("Maximum"),
			aws.String("Average"),
			aws.String("Minimum"),
		},
		Dimensions: []*cloudwatch.Dimension{
			{
				Name:  aws.String("ClusterName"),
				Value: aws.String(cluster),
			},
		},
	}

	resp, err := models.CloudWatch().GetMetricStatistics(params)

	if err != nil {
		RenderError(rw, err)
		return
	}

	RenderJson(rw, resp)
}
Ejemplo n.º 2
0
func startClusterMonitor() {
	var log = logger.New("ns=cluster_monitor")

Tick:
	for _ = range time.Tick(5 * time.Minute) {
		log.Log("tick")

		// Ger Rack InstanceCount Parameter
		instanceCount := 0

		res, err := models.CloudFormation().DescribeStacks(
			&cloudformation.DescribeStacksInput{
				StackName: aws.String(os.Getenv("RACK")),
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		for _, p := range res.Stacks[0].Parameters {
			if *p.ParameterKey == "InstanceCount" {
				c, err := strconv.Atoi(*p.ParameterValue)

				if err != nil {
					log.Error(err)
					break Tick
				}

				instanceCount = c
				break
			}
		}

		// List and Describe ECS Container Instances
		ires, err := models.ECS().ListContainerInstances(
			&ecs.ListContainerInstancesInput{
				Cluster: aws.String(os.Getenv("CLUSTER")),
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		dres, err := models.ECS().DescribeContainerInstances(
			&ecs.DescribeContainerInstancesInput{
				Cluster:            aws.String(os.Getenv("CLUSTER")),
				ContainerInstances: ires.ContainerInstanceARNs,
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		cInstanceIds := make([]string, 0)
		cInstanceConnections := make(map[string]bool)

		for _, i := range dres.ContainerInstances {
			cInstanceConnections[*i.EC2InstanceID] = *i.AgentConnected

			if *i.AgentConnected {
				cInstanceIds = append(cInstanceIds, *i.EC2InstanceID)
			}
		}

		// Get and Describe Rack ASG Resource
		resources, err := models.ListResources(os.Getenv("RACK"))

		ares, err := models.AutoScaling().DescribeAutoScalingGroups(
			&autoscaling.DescribeAutoScalingGroupsInput{
				AutoScalingGroupNames: []*string{
					aws.String(resources["Instances"].Id),
				},
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		// Test if ASG Instance is registered and connected in ECS cluster

		aInstanceIds := []string{}
		uInstanceIds := []string{}

		for _, i := range ares.AutoScalingGroups[0].Instances {
			if connected, exists := cInstanceConnections[*i.InstanceID]; connected && exists {
				aInstanceIds = append(aInstanceIds, *i.InstanceID)
			} else {
				// Not registered or not connected => set Unhealthy
				if *i.LifecycleState == "InService" {
					_, err := models.AutoScaling().SetInstanceHealth(
						&autoscaling.SetInstanceHealthInput{
							HealthStatus:             aws.String("Unhealthy"),
							InstanceID:               aws.String(*i.InstanceID),
							ShouldRespectGracePeriod: aws.Boolean(true),
						},
					)

					if err != nil {
						log.Error(err)
						continue
					}

					uInstanceIds = append(uInstanceIds, *i.InstanceID)
				}
			}
		}

		sort.Strings(aInstanceIds)
		sort.Strings(cInstanceIds)
		sort.Strings(uInstanceIds)

		log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
	}
}
Ejemplo n.º 3
0
func SystemUpdate(rw http.ResponseWriter, r *http.Request) {
	log := systemLogger("update").Start()

	app, err := models.GetApp(os.Getenv("RACK"))

	if err != nil {
		log.Error(err)
		RenderError(rw, err)
		return
	}

	p := map[string]string{}

	if version := GetForm(r, "version"); version != "" {
		p["Version"] = version
	}

	if count := GetForm(r, "count"); count != "" {
		p["InstanceCount"] = count
	}

	if t := GetForm(r, "type"); t != "" {
		p["InstanceType"] = t
	}

	if len(p) > 0 {
		req := &cloudformation.UpdateStackInput{
			StackName:    aws.String(app.Name),
			Capabilities: []*string{aws.String("CAPABILITY_IAM")},
		}

		if p["Version"] == "" {
			req.UsePreviousTemplate = aws.Boolean(true)
		} else {
			req.TemplateURL = aws.String(fmt.Sprintf("http://convox.s3.amazonaws.com/release/%s/formation.json", p["Version"]))
		}

		params := app.Parameters

		for key, val := range p {
			params[key] = val
		}

		for key, val := range params {
			req.Parameters = append(req.Parameters, &cloudformation.Parameter{
				ParameterKey:   aws.String(key),
				ParameterValue: aws.String(val),
			})
		}

		_, err := models.CloudFormation().UpdateStack(req)

		if ae, ok := err.(awserr.Error); ok {
			if ae.Code() == "ValidationError" {
				switch {
				case strings.Index(ae.Error(), "No updates are to be performed") > -1:
					RenderNotFound(rw, fmt.Sprintf("no system updates are to be performed."))
					return
				case strings.Index(ae.Error(), "can not be updated") > -1:
					RenderNotFound(rw, fmt.Sprintf("system is already updating."))
					return
				}
			}
		}

		if err != nil {
			log.Error(err)
			RenderError(rw, err)
			return
		}
	}

	Redirect(rw, r, "/system")
}