func ClusterTop(rw http.ResponseWriter, r *http.Request) {
name := aws.String(os.Getenv("RACK"))
res, err := models.CloudFormation().DescribeStacks(&cloudformation.DescribeStacksInput{StackName: name})
if err != nil {
RenderError(rw, err)
return
}
if len(res.Stacks) == 0 {
RenderError(rw, fmt.Errorf("Stack %s does not exist", os.Getenv("RACK")))
return
}
stack := res.Stacks[0]
outputs := make(map[string]string)
for _, output := range stack.Outputs {
outputs[*output.OutputKey] = *output.OutputValue
}
cluster := outputs["Cluster"]
params := &cloudwatch.GetMetricStatisticsInput{
MetricName: aws.String(mux.Vars(r)["metric"]),
StartTime: aws.Time(time.Now().Add(-2 * time.Minute)),
EndTime: aws.Time(time.Now()),
Period: aws.Long(60),
Namespace: aws.String("AWS/ECS"),
Statistics: []*string{ // Required
aws.String("Maximum"),
aws.String("Average"),
aws.String("Minimum"),
},
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String("ClusterName"),
Value: aws.String(cluster),
},
},
}
resp, err := models.CloudWatch().GetMetricStatistics(params)
if err != nil {
RenderError(rw, err)
return
}
RenderJson(rw, resp)
}
func startClusterMonitor() {
var log = logger.New("ns=cluster_monitor")
Tick:
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
// Ger Rack InstanceCount Parameter
instanceCount := 0
res, err := models.CloudFormation().DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String(os.Getenv("RACK")),
},
)
if err != nil {
log.Error(err)
continue
}
for _, p := range res.Stacks[0].Parameters {
if *p.ParameterKey == "InstanceCount" {
c, err := strconv.Atoi(*p.ParameterValue)
if err != nil {
log.Error(err)
break Tick
}
instanceCount = c
break
}
}
// List and Describe ECS Container Instances
ires, err := models.ECS().ListContainerInstances(
&ecs.ListContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
},
)
if err != nil {
log.Error(err)
continue
}
dres, err := models.ECS().DescribeContainerInstances(
&ecs.DescribeContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
ContainerInstances: ires.ContainerInstanceARNs,
},
)
if err != nil {
log.Error(err)
continue
}
cInstanceIds := make([]string, 0)
cInstanceConnections := make(map[string]bool)
for _, i := range dres.ContainerInstances {
cInstanceConnections[*i.EC2InstanceID] = *i.AgentConnected
if *i.AgentConnected {
cInstanceIds = append(cInstanceIds, *i.EC2InstanceID)
}
}
// Get and Describe Rack ASG Resource
resources, err := models.ListResources(os.Getenv("RACK"))
ares, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
aInstanceIds := []string{}
uInstanceIds := []string{}
for _, i := range ares.AutoScalingGroups[0].Instances {
if connected, exists := cInstanceConnections[*i.InstanceID]; connected && exists {
aInstanceIds = append(aInstanceIds, *i.InstanceID)
} else {
// Not registered or not connected => set Unhealthy
if *i.LifecycleState == "InService" {
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceID: aws.String(*i.InstanceID),
ShouldRespectGracePeriod: aws.Boolean(true),
},
)
if err != nil {
log.Error(err)
continue
}
uInstanceIds = append(uInstanceIds, *i.InstanceID)
}
}
}
sort.Strings(aInstanceIds)
sort.Strings(cInstanceIds)
sort.Strings(uInstanceIds)
log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
}
}
func SystemUpdate(rw http.ResponseWriter, r *http.Request) {
log := systemLogger("update").Start()
app, err := models.GetApp(os.Getenv("RACK"))
if err != nil {
log.Error(err)
RenderError(rw, err)
return
}
p := map[string]string{}
if version := GetForm(r, "version"); version != "" {
p["Version"] = version
}
if count := GetForm(r, "count"); count != "" {
p["InstanceCount"] = count
}
if t := GetForm(r, "type"); t != "" {
p["InstanceType"] = t
}
if len(p) > 0 {
req := &cloudformation.UpdateStackInput{
StackName: aws.String(app.Name),
Capabilities: []*string{aws.String("CAPABILITY_IAM")},
}
if p["Version"] == "" {
req.UsePreviousTemplate = aws.Boolean(true)
} else {
req.TemplateURL = aws.String(fmt.Sprintf("http://convox.s3.amazonaws.com/release/%s/formation.json", p["Version"]))
}
params := app.Parameters
for key, val := range p {
params[key] = val
}
for key, val := range params {
req.Parameters = append(req.Parameters, &cloudformation.Parameter{
ParameterKey: aws.String(key),
ParameterValue: aws.String(val),
})
}
_, err := models.CloudFormation().UpdateStack(req)
if ae, ok := err.(awserr.Error); ok {
if ae.Code() == "ValidationError" {
switch {
case strings.Index(ae.Error(), "No updates are to be performed") > -1:
RenderNotFound(rw, fmt.Sprintf("no system updates are to be performed."))
return
case strings.Index(ae.Error(), "can not be updated") > -1:
RenderNotFound(rw, fmt.Sprintf("system is already updating."))
return
}
}
}
if err != nil {
log.Error(err)
RenderError(rw, err)
return
}
}
Redirect(rw, r, "/system")
}