forked from bgentry/kernel
/
cluster_monitor.go
141 lines (112 loc) · 3.44 KB
/
cluster_monitor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
package main
import (
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/convox/kernel/models"
"github.com/convox/kernel/Godeps/_workspace/src/github.com/ddollar/logger"
"github.com/convox/kernel/Godeps/_workspace/src/github.com/awslabs/aws-sdk-go/aws"
"github.com/convox/kernel/Godeps/_workspace/src/github.com/awslabs/aws-sdk-go/service/autoscaling"
"github.com/convox/kernel/Godeps/_workspace/src/github.com/awslabs/aws-sdk-go/service/cloudformation"
"github.com/convox/kernel/Godeps/_workspace/src/github.com/awslabs/aws-sdk-go/service/ecs"
)
func startClusterMonitor() {
var log = logger.New("ns=cluster_monitor")
Tick:
for _ = range time.Tick(5 * time.Minute) {
log.Log("tick")
// Ger Rack InstanceCount Parameter
instanceCount := 0
res, err := models.CloudFormation().DescribeStacks(
&cloudformation.DescribeStacksInput{
StackName: aws.String(os.Getenv("RACK")),
},
)
if err != nil {
log.Error(err)
continue
}
for _, p := range res.Stacks[0].Parameters {
if *p.ParameterKey == "InstanceCount" {
c, err := strconv.Atoi(*p.ParameterValue)
if err != nil {
log.Error(err)
break Tick
}
instanceCount = c
break
}
}
// List and Describe ECS Container Instances
ires, err := models.ECS().ListContainerInstances(
&ecs.ListContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
},
)
if err != nil {
log.Error(err)
continue
}
dres, err := models.ECS().DescribeContainerInstances(
&ecs.DescribeContainerInstancesInput{
Cluster: aws.String(os.Getenv("CLUSTER")),
ContainerInstances: ires.ContainerInstanceARNs,
},
)
if err != nil {
log.Error(err)
continue
}
cInstanceIds := make([]string, 0)
cInstanceConnections := make(map[string]bool)
for _, i := range dres.ContainerInstances {
cInstanceConnections[*i.EC2InstanceID] = *i.AgentConnected
if *i.AgentConnected {
cInstanceIds = append(cInstanceIds, *i.EC2InstanceID)
}
}
// Get and Describe Rack ASG Resource
resources, err := models.ListResources(os.Getenv("RACK"))
ares, err := models.AutoScaling().DescribeAutoScalingGroups(
&autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{
aws.String(resources["Instances"].Id),
},
},
)
if err != nil {
log.Error(err)
continue
}
// Test if ASG Instance is registered and connected in ECS cluster
aInstanceIds := []string{}
uInstanceIds := []string{}
for _, i := range ares.AutoScalingGroups[0].Instances {
if connected, exists := cInstanceConnections[*i.InstanceID]; connected && exists {
aInstanceIds = append(aInstanceIds, *i.InstanceID)
} else {
// Not registered or not connected => set Unhealthy
if *i.LifecycleState == "InService" {
_, err := models.AutoScaling().SetInstanceHealth(
&autoscaling.SetInstanceHealthInput{
HealthStatus: aws.String("Unhealthy"),
InstanceID: aws.String(*i.InstanceID),
ShouldRespectGracePeriod: aws.Boolean(true),
},
)
if err != nil {
log.Error(err)
continue
}
uInstanceIds = append(uInstanceIds, *i.InstanceID)
}
}
}
sort.Strings(aInstanceIds)
sort.Strings(cInstanceIds)
sort.Strings(uInstanceIds)
log.Log("InstanceCount=%v connected='%v' healthy='%v' marked='%s'", instanceCount, strings.Join(cInstanceIds, ","), strings.Join(aInstanceIds, ","), strings.Join(uInstanceIds, ","))
}
}