func (instances Instances) describeASG() error {
resources, err := models.ListResources(os.Getenv("RACK"))
res, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
return err
}
for _, i := range res.AutoScalingGroups[0].Instances {
instance := instances[*i.InstanceId]
instance.Id = *i.InstanceId
instance.ASG = *i.LifecycleState == "InService"
instances[*i.InstanceId] = instance
}
return nil
}
func (instances Instances) describeASG() error {
resources, err := models.ListResources(os.Getenv("RACK"))
res, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
return err
}
for _, i := range res.AutoScalingGroups[0].Instances {
instance := instances[*i.InstanceId]
instance.Id = *i.InstanceId
instance.ASG = *i.LifecycleState == "InService"
instances[*i.InstanceId] = instance
}
// describe and log every recent ASG activity
dres, err := models.AutoScaling().DescribeScalingActivities(
&autoscaling.DescribeScalingActivitiesInput{
AutoScalingGroupName: aws.String(resources["Instances"].Id),
},
)
if err != nil {
return nil
}
for _, a := range dres.Activities {
if lastASGActivity.Before(*a.StartTime) {
fmt.Printf("who=\"EC2/ASG\" what=%q why=%q\n", *a.Description, *a.Cause)
lastASGActivity = *a.StartTime
}
}
return nil
}
func BuildCreate(rw http.ResponseWriter, r *http.Request) error {
build := models.NewBuild(mux.Vars(r)["app"])
err := r.ParseMultipartForm(50 * 1024 * 1024)
if err != nil && err != http.ErrNotMultipart {
return err
}
err = build.Save()
if err != nil {
return err
}
resources, err := models.ListResources(os.Getenv("RACK"))
if err != nil {
return err
}
ch := make(chan error)
source, _, err := r.FormFile("source")
if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
return err
}
if source != nil {
err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)
if err != nil {
return err
}
go build.ExecuteLocal(source, ch)
err = <-ch
if err != nil {
return err
} else {
return RenderJson(rw, build)
}
}
if repo := r.FormValue("repo"); repo != "" {
go build.ExecuteRemote(repo, ch)
err = <-ch
if err != nil {
return err
} else {
return RenderJson(rw, build)
}
}
return fmt.Errorf("no source or repo")
}
func StartCluster() {
var log = logger.New("ns=cluster_monitor")
defer recoverWith(func(err error) {
helpers.Error(log, err)
})
Tick:
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
// Ger Rack InstanceCount Parameter
instanceCount := 0
// instanceType := "unknown"
res, err := models.CloudFormation().DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String(os.Getenv("RACK")),
},
)
if err != nil {
log.Error(err)
continue
}
for _, p := range res.Stacks[0].Parameters {
if *p.ParameterKey == "InstanceCount" {
c, err := strconv.Atoi(*p.ParameterValue)
if err != nil {
log.Error(err)
break Tick
}
instanceCount = c
}
// if *p.ParameterKey == "InstanceType" {
// instanceType = *p.ParameterValue
// }
}
// helpers.SendMixpanelEvent("kernel-cluster-monitor", fmt.Sprintf("count=%d type=%s", instanceCount, instanceType))
// List and Describe ECS Container Instances
ires, err := models.ECS().ListContainerInstances(
&ecs.ListContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
},
)
if err != nil {
log.Error(err)
continue
}
dres, err := models.ECS().DescribeContainerInstances(
&ecs.DescribeContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
ContainerInstances: ires.ContainerInstanceArns,
},
)
if err != nil {
log.Error(err)
continue
}
cInstanceIds := make([]string, 0)
cInstanceConnections := make(map[string]bool)
for _, i := range dres.ContainerInstances {
cInstanceConnections[*i.Ec2InstanceId] = *i.AgentConnected
if *i.AgentConnected {
cInstanceIds = append(cInstanceIds, *i.Ec2InstanceId)
}
}
// Get and Describe Rack ASG Resource
resources, err := models.ListResources(os.Getenv("RACK"))
ares, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
aInstanceIds := []string{}
uInstanceIds := []string{}
for _, i := range ares.AutoScalingGroups[0].Instances {
if connected, exists := cInstanceConnections[*i.InstanceId]; connected && exists {
aInstanceIds = append(aInstanceIds, *i.InstanceId)
} else {
// Not registered or not connected => set Unhealthy
if *i.LifecycleState == "InService" {
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceId: aws.String(*i.InstanceId),
ShouldRespectGracePeriod: aws.Bool(true),
},
)
if err != nil {
log.Error(err)
continue
}
uInstanceIds = append(uInstanceIds, *i.InstanceId)
}
}
}
sort.Strings(aInstanceIds)
sort.Strings(cInstanceIds)
sort.Strings(uInstanceIds)
// if len(uInstanceIds) > 0 {
// helpers.SendMixpanelEvent("kernel-cluster-monitor-mark", strings.Join(uInstanceIds, ","))
// }
log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
}
}
func BuildCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error {
build := models.NewBuild(mux.Vars(r)["app"])
build.Description = r.FormValue("description")
manifest := r.FormValue("manifest") // empty value will default "docker-compose.yml" in cmd/build
// use deprecated "config" param if set and "manifest" is not
if config := r.FormValue("config"); config != "" && manifest == "" {
manifest = config
}
if build.IsRunning() {
err := fmt.Errorf("Another build is currently running. Please try again later.")
helpers.TrackError("build", err, map[string]interface{}{"at": "build.IsRunning"})
return httperr.Server(err)
}
err := r.ParseMultipartForm(50 * 1024 * 1024)
if err != nil && err != http.ErrNotMultipart {
helpers.TrackError("build", err, map[string]interface{}{"at": "ParseMultipartForm"})
return httperr.Server(err)
}
err = build.Save()
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "build.Save"})
return httperr.Server(err)
}
resources, err := models.ListResources(os.Getenv("RACK"))
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "models.ListResources"})
return httperr.Server(err)
}
ch := make(chan error)
source, _, err := r.FormFile("source")
if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
helpers.TrackError("build", err, map[string]interface{}{"at": "FormFile"})
return httperr.Server(err)
}
cache := !(r.FormValue("cache") == "false")
if source != nil {
err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "models.S3PutFile"})
return httperr.Server(err)
}
go build.ExecuteLocal(source, cache, manifest, ch)
err = <-ch
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "models.ExecuteLocal"})
return httperr.Server(err)
} else {
return RenderJson(rw, build)
}
}
if repo := r.FormValue("repo"); repo != "" {
go build.ExecuteRemote(repo, cache, manifest, ch)
err = <-ch
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteRemote"})
return httperr.Server(err)
} else {
return RenderJson(rw, build)
}
}
if data := r.FormValue("index"); data != "" {
var index models.Index
err := json.Unmarshal([]byte(data), &index)
if err != nil {
return httperr.Server(err)
}
go build.ExecuteIndex(index, cache, manifest, ch)
err = <-ch
if err != nil {
helpers.TrackError("build", err, map[string]interface{}{"at": "build.ExecuteIndex"})
return httperr.Server(err)
} else {
return RenderJson(rw, build)
}
}
return httperr.Errorf(403, "no source or repo")
}
func BuildCreate(rw http.ResponseWriter, r *http.Request) *httperr.Error {
build := models.NewBuild(mux.Vars(r)["app"])
config := r.FormValue("config")
if build.IsRunning() {
return httperr.Errorf(403, "another build is currently running. Please try again later.")
}
err := r.ParseMultipartForm(50 * 1024 * 1024)
if err != nil && err != http.ErrNotMultipart {
return httperr.Server(err)
}
err = build.Save()
if err != nil {
return httperr.Server(err)
}
resources, err := models.ListResources(os.Getenv("RACK"))
if err != nil {
return httperr.Server(err)
}
ch := make(chan error)
source, _, err := r.FormFile("source")
if err != nil && err != http.ErrMissingFile && err != http.ErrNotMultipart {
return httperr.Server(err)
}
cache := !(r.FormValue("cache") == "false")
if source != nil {
err = models.S3PutFile(resources["RegistryBucket"].Id, fmt.Sprintf("builds/%s.tgz", build.Id), source, false)
if err != nil {
return httperr.Server(err)
}
go build.ExecuteLocal(source, cache, config, ch)
err = <-ch
if err != nil {
return httperr.Server(err)
} else {
return RenderJson(rw, build)
}
}
if repo := r.FormValue("repo"); repo != "" {
go build.ExecuteRemote(repo, cache, config, ch)
err = <-ch
if err != nil {
return httperr.Server(err)
} else {
return RenderJson(rw, build)
}
}
return httperr.Errorf(403, "no source or repo")
}
