Example #1
0
func (instances Instances) describeASG() error {
	resources, err := models.ListResources(os.Getenv("RACK"))

	res, err := models.AutoScaling().DescribeAutoScalingGroups(
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: []*string{
				aws.String(resources["Instances"].Id),
			},
		},
	)

	if err != nil {
		return err
	}

	for _, i := range res.AutoScalingGroups[0].Instances {
		instance := instances[*i.InstanceId]

		instance.Id = *i.InstanceId
		instance.ASG = *i.LifecycleState == "InService"

		instances[*i.InstanceId] = instance
	}

	return nil
}
Example #2
0
func (instances Instances) describeASG() error {
	resources, err := models.ListResources(os.Getenv("RACK"))

	res, err := models.AutoScaling().DescribeAutoScalingGroups(
		&autoscaling.DescribeAutoScalingGroupsInput{
			AutoScalingGroupNames: []*string{
				aws.String(resources["Instances"].Id),
			},
		},
	)
	if err != nil {
		return err
	}

	for _, i := range res.AutoScalingGroups[0].Instances {
		instance := instances[*i.InstanceId]

		instance.Id = *i.InstanceId
		instance.ASG = *i.LifecycleState == "InService"

		instances[*i.InstanceId] = instance
	}

	// describe and log every recent ASG activity
	dres, err := models.AutoScaling().DescribeScalingActivities(
		&autoscaling.DescribeScalingActivitiesInput{
			AutoScalingGroupName: aws.String(resources["Instances"].Id),
		},
	)
	if err != nil {
		return nil
	}

	for _, a := range dres.Activities {
		if lastASGActivity.Before(*a.StartTime) {
			fmt.Printf("who=\"EC2/ASG\" what=%q why=%q\n", *a.Description, *a.Cause)
			lastASGActivity = *a.StartTime
		}
	}

	return nil
}
Example #3
0
func BuildCreate(rw http.ResponseWriter, r *http.Request) error {
	build := models.NewBuild(mux.Vars(r)["app"])

	err := r.ParseMultipartForm(50 * 1024 * 1024)

	if err != nil && err != http.ErrNotMultipart {
		return err
	}

	err = build.Save()

	if err != nil {
		return err
	}

	resources, err := models.ListResources(os.Getenv("RACK"))

	if err != nil {
		return err
	}

	ch := make(chan error)

	source, _, err := r.FormFile("source")

	if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
		return err
	}

	if source != nil {
		err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)

		if err != nil {
			return err
		}

		go build.ExecuteLocal(source, ch)

		err = <-ch

		if err != nil {
			return err
		} else {
			return RenderJson(rw, build)
		}
	}

	if repo := r.FormValue("repo"); repo != "" {
		go build.ExecuteRemote(repo, ch)

		err = <-ch

		if err != nil {
			return err
		} else {
			return RenderJson(rw, build)
		}
	}

	return fmt.Errorf("no source or repo")
}
Example #4
0
func StartCluster() {
	var log = logger.New("ns=cluster_monitor")

	defer recoverWith(func(err error) {
		helpers.Error(log, err)
	})

Tick:
	for _ = range time.Tick(5 * time.Minute) {
		log.Log("tick")

		// Ger Rack InstanceCount Parameter
		instanceCount := 0
		// instanceType := "unknown"

		res, err := models.CloudFormation().DescribeStacks(
			&cloudformation.DescribeStacksInput{
				StackName: aws.String(os.Getenv("RACK")),
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		for _, p := range res.Stacks[0].Parameters {
			if *p.ParameterKey == "InstanceCount" {
				c, err := strconv.Atoi(*p.ParameterValue)

				if err != nil {
					log.Error(err)
					break Tick
				}

				instanceCount = c
			}

			// if *p.ParameterKey == "InstanceType" {
			//   instanceType = *p.ParameterValue
			// }
		}

		// helpers.SendMixpanelEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", instanceCount, instanceType))

		// List and Describe ECS Container Instances
		ires, err := models.ECS().ListContainerInstances(
			&ecs.ListContainerInstancesInput{
				Cluster: aws.String(os.Getenv("CLUSTER")),
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		dres, err := models.ECS().DescribeContainerInstances(
			&ecs.DescribeContainerInstancesInput{
				Cluster:            aws.String(os.Getenv("CLUSTER")),
				ContainerInstances: ires.ContainerInstanceArns,
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		cInstanceIds := make([]string, 0)
		cInstanceConnections := make(map[string]bool)

		for _, i := range dres.ContainerInstances {
			cInstanceConnections[*i.Ec2InstanceId] = *i.AgentConnected

			if *i.AgentConnected {
				cInstanceIds = append(cInstanceIds, *i.Ec2InstanceId)
			}
		}

		// Get and Describe Rack ASG Resource
		resources, err := models.ListResources(os.Getenv("RACK"))

		ares, err := models.AutoScaling().DescribeAutoScalingGroups(
			&autoscaling.DescribeAutoScalingGroupsInput{
				AutoScalingGroupNames: []*string{
					aws.String(resources["Instances"].Id),
				},
			},
		)

		if err != nil {
			log.Error(err)
			continue
		}

		// Test if ASG Instance is registered and connected in ECS cluster

		aInstanceIds := []string{}
		uInstanceIds := []string{}

		for _, i := range ares.AutoScalingGroups[0].Instances {
			if connected, exists := cInstanceConnections[*i.InstanceId]; connected && exists {
				aInstanceIds = append(aInstanceIds, *i.InstanceId)
			} else {
				// Not registered or not connected => set Unhealthy
				if *i.LifecycleState == "InService" {
					_, err := models.AutoScaling().SetInstanceHealth(
						&autoscaling.SetInstanceHealthInput{
							HealthStatus:             aws.String("Unhealthy"),
							InstanceId:               aws.String(*i.InstanceId),
							ShouldRespectGracePeriod: aws.Bool(true),
						},
					)

					if err != nil {
						log.Error(err)
						continue
					}

					uInstanceIds = append(uInstanceIds, *i.InstanceId)
				}
			}
		}

		sort.Strings(aInstanceIds)
		sort.Strings(cInstanceIds)
		sort.Strings(uInstanceIds)

		// if len(uInstanceIds) > 0 {
		//   helpers.SendMixpanelEvent("kernel-cluster-monitor-mark", strings.Join(uInstanceIds, ","))
		// }

		log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
	}
}
Example #5
0
func BuildCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error {
	build := models.NewBuild(mux.Vars(r)["app"])
	build.Description = r.FormValue("description")

	manifest := r.FormValue("manifest") // empty value will default "docker-compose.yml" in cmd/build

	// use deprecated "config" param if set and "manifest" is not
	if config := r.FormValue("config"); config != "" && manifest == "" {
		manifest = config
	}

	if build.IsRunning() {
		err := fmt.Errorf("Another build is currently running. Please try again later.")
		helpers.TrackError("build", err, map[string]interface{}{"at": "build.IsRunning"})
		return httperr.Server(err)
	}

	err := r.ParseMultipartForm(50 * 1024 * 1024)

	if err != nil && err != http.ErrNotMultipart {
		helpers.TrackError("build", err, map[string]interface{}{"at": "ParseMultipartForm"})
		return httperr.Server(err)
	}

	err = build.Save()

	if err != nil {
		helpers.TrackError("build", err, map[string]interface{}{"at": "build.Save"})
		return httperr.Server(err)
	}

	resources, err := models.ListResources(os.Getenv("RACK"))

	if err != nil {
		helpers.TrackError("build", err, map[string]interface{}{"at": "models.ListResources"})
		return httperr.Server(err)
	}

	ch := make(chan error)

	source, _, err := r.FormFile("source")

	if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
		helpers.TrackError("build", err, map[string]interface{}{"at": "FormFile"})
		return httperr.Server(err)
	}

	cache := !(r.FormValue("cache") == "false")

	if source != nil {
		err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)

		if err != nil {
			helpers.TrackError("build", err, map[string]interface{}{"at": "models.S3PutFile"})
			return httperr.Server(err)
		}

		go build.ExecuteLocal(source, cache, manifest, ch)

		err = <-ch

		if err != nil {
			helpers.TrackError("build", err, map[string]interface{}{"at": "models.ExecuteLocal"})
			return httperr.Server(err)
		} else {
			return RenderJson(rw, build)
		}
	}

	if repo := r.FormValue("repo"); repo != "" {
		go build.ExecuteRemote(repo, cache, manifest, ch)

		err = <-ch

		if err != nil {
			helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteRemote"})
			return httperr.Server(err)
		} else {
			return RenderJson(rw, build)
		}
	}

	if data := r.FormValue("index"); data != "" {
		var index models.Index

		err := json.Unmarshal([]byte(data), &index)

		if err != nil {
			return httperr.Server(err)
		}

		go build.ExecuteIndex(index, cache, manifest, ch)

		err = <-ch

		if err != nil {
			helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteIndex"})
			return httperr.Server(err)
		} else {
			return RenderJson(rw, build)
		}
	}

	return httperr.Errorf(403, "no source or repo")
}
Example #6
0
func BuildCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error {
	build := models.NewBuild(mux.Vars(r)["app"])
	config := r.FormValue("config")

	if build.IsRunning() {
		return httperr.Errorf(403, "another build is currently running. Please try again later.")
	}

	err := r.ParseMultipartForm(50 * 1024 * 1024)

	if err != nil && err != http.ErrNotMultipart {
		return httperr.Server(err)
	}

	err = build.Save()

	if err != nil {
		return httperr.Server(err)
	}

	resources, err := models.ListResources(os.Getenv("RACK"))

	if err != nil {
		return httperr.Server(err)
	}

	ch := make(chan error)

	source, _, err := r.FormFile("source")

	if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
		return httperr.Server(err)
	}

	cache := !(r.FormValue("cache") == "false")

	if source != nil {
		err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)

		if err != nil {
			return httperr.Server(err)
		}

		go build.ExecuteLocal(source, cache, config, ch)

		err = <-ch

		if err != nil {
			return httperr.Server(err)
		} else {
			return RenderJson(rw, build)
		}
	}

	if repo := r.FormValue("repo"); repo != "" {
		go build.ExecuteRemote(repo, cache, config, ch)

		err = <-ch

		if err != nil {
			return httperr.Server(err)
		} else {
			return RenderJson(rw, build)
		}
	}

	return httperr.Errorf(403, "no source or repo")
}