func awsGetELBHostCounts(cw cloudwatch.CloudWatch, md *opentsdb.MultiDataPoint, loadBalancer *elb.LoadBalancerDescription) error {
search := cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(time.Now().UTC().Add(time.Second * -60)),
EndTime: aws.Time(time.Now().UTC()),
MetricName: aws.String("HealthyHostCount"),
Period: &aws_period,
Statistics: []*string{aws.String("Average")},
Namespace: aws.String("AWS/ELB"),
Unit: aws.String("Count"),
Dimensions: []*cloudwatch.Dimension{{Name: aws.String("LoadBalancerName"), Value: loadBalancer.LoadBalancerName}},
}
resp, err := cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsELBHostsHealthy, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Count, descAWSELBHostCount)
}
search.MetricName = aws.String("UnhealthyHostCount")
resp, err = cw.GetMetricStatistics(&search)
if err != nil {
return err
}
if resp.Datapoints == nil {
AddTS(md, awsELBHostsUnHealthy, time.Now().UTC().Unix(), 0, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Count, descAWSELBHostCount)
} else {
for _, datapoint := range resp.Datapoints {
AddTS(md, awsELBHostsUnHealthy, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Count, descAWSELBHostCount)
}
}
return nil
}
// VerifyMetrics whether the response is as expected
// the expected value can be 0 or positive
func VerifyMetrics(cwclient *cloudwatch.CloudWatch, params *cloudwatch.GetMetricStatisticsInput, idleCluster bool) error {
resp, err := cwclient.GetMetricStatistics(params)
if err != nil {
return fmt.Errorf("Error getting metrics of cluster: %v", err)
}
if resp == nil || resp.Datapoints == nil {
return fmt.Errorf("Cloudwatch get metrics failed, returned null")
}
metricsCount := len(resp.Datapoints)
if metricsCount == 0 {
return fmt.Errorf("No datapoints returned")
}
datapoint := resp.Datapoints[metricsCount-1]
// Samplecount is always expected to be "1" for cluster metrics
if *datapoint.SampleCount != 1.0 {
return fmt.Errorf("Incorrect SampleCount %f, expected 1", *datapoint.SampleCount)
}
if idleCluster {
if *datapoint.Average != 0.0 {
return fmt.Errorf("non-zero utilization for idle cluster")
}
} else {
if *datapoint.Average == 0.0 {
return fmt.Errorf("utilization is zero for non-idle cluster")
}
}
return nil
}
func awsGetDiskOps(cw cloudwatch.CloudWatch, md *opentsdb.MultiDataPoint, instance *ec2.Instance) error {
search := cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(time.Now().UTC().Add(time.Second * -600)),
EndTime: aws.Time(time.Now().UTC()),
MetricName: aws.String("DiskReadOps"),
Period: &aws_period,
Statistics: []*string{aws.String("Average")},
Namespace: aws.String("AWS/EC2"),
Unit: aws.String("Count"),
Dimensions: []*cloudwatch.Dimension{{Name: aws.String("InstanceId"), Value: instance.InstanceId}},
}
resp, err := cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsEC2DiskOps, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"instance": *instance.InstanceId, "operation": "read"}, metadata.Gauge, metadata.Count, descAWSEC2DiskOps)
}
search.MetricName = aws.String("DiskWriteOps")
resp, err = cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsEC2DiskOps, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"instance": *instance.InstanceId, "operation": "write"}, metadata.Gauge, metadata.Count, descAWSEC2DiskOps)
}
return nil
}
func getLastPoint(cloudWatch *cloudwatch.CloudWatch, dimension *cloudwatch.Dimension, metricName string) (float64, error) {
now := time.Now()
response, err := cloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(now.Add(time.Duration(600) * time.Second * -1)),
EndTime: aws.Time(now),
Period: aws.Int64(60),
Namespace: aws.String("AWS/EC2"),
Dimensions: []*cloudwatch.Dimension{dimension},
MetricName: aws.String(metricName),
Statistics: []*string{aws.String("Average")},
})
if err != nil {
return 0, err
}
datapoints := response.Datapoints
if len(datapoints) == 0 {
return 0, errors.New("fetched no datapoints")
}
latest := time.Unix(0, 0)
var latestVal float64
for _, dp := range datapoints {
if dp.Timestamp.Before(latest) {
continue
}
latest = *dp.Timestamp
latestVal = *dp.Average
}
return latestVal, nil
}
// Helper function to send data.
func Send(cw *cloudwatch.CloudWatch, dn, dv, n string, v float64) {
_, err := cw.PutMetricData(&cloudwatch.PutMetricDataInput{
MetricData: []*cloudwatch.MetricDatum{
{
MetricName: aws.String(n),
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String(dn),
Value: aws.String(dv),
},
},
Timestamp: aws.Time(time.Now()),
Value: aws.Float64(v),
},
},
Namespace: aws.String("Custom"),
})
if err != nil {
log.Println(err)
return
}
log.WithFields(log.Fields{
"DimensionName": dn,
"DimensionValue": dv,
"MetricName": n,
"MetricValue": v,
}).Info("Successfully posted to CloudWatch")
}
// Grabs datapoints from CloudWatch API and writes them to disk
func getDataPoints(metric cloudwatch.Metric, svc *cloudwatch.CloudWatch, wg *sync.WaitGroup, startTime, endTime time.Time) {
// Signal WaitGroup
defer wg.Done()
// Set search parameters
params := &cloudwatch.GetMetricStatisticsInput{
EndTime: aws.Time(endTime),
MetricName: metric.MetricName,
Namespace: metric.Namespace,
Period: period,
StartTime: aws.Time(startTime),
Statistics: []*string{
aws.String("SampleCount"),
aws.String("Average"),
aws.String("Sum"),
aws.String("Minimum"),
aws.String("Maximum"),
},
Dimensions: metric.Dimensions,
Unit: aws.String("Seconds"),
}
// use metric to query GetMetricStatistics
resp, err := svc.GetMetricStatistics(params)
check(err)
// Check for data points
if resp.Datapoints != nil {
// Build directory structure
var filename string
var dirname string
if metric.Dimensions != nil {
filename = fmt.Sprintf("%s/%s/%s/%s", *baseDir, *metric.Namespace, *metric.Dimensions[0].Name, *metric.Dimensions[0].Value)
dirname = fmt.Sprintf("%s/%s/%s", *baseDir, *metric.Namespace, *metric.Dimensions[0].Name)
} else {
filename = fmt.Sprintf("%s/%s/%s", *baseDir, *metric.Namespace, *metric.MetricName)
dirname = fmt.Sprintf("%s/%s", *baseDir, *metric.Namespace)
}
// Create any missing directories
err := os.MkdirAll(dirname, 0755)
check(err)
// Open/create file for writing/appending
fmt.Printf(strings.Repeat(".", len(resp.Datapoints)))
json, err := json.Marshal(resp)
check(err)
f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
check(err)
defer f.Close()
_, err = f.Write(json)
}
}
func awsGetELBLatency(cw cloudwatch.CloudWatch, md *opentsdb.MultiDataPoint, loadBalancer *elb.LoadBalancerDescription) error {
search := cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(time.Now().UTC().Add(time.Second * -4000)),
EndTime: aws.Time(time.Now().UTC()),
MetricName: aws.String("Latency"),
Period: &aws_period,
Statistics: []*string{aws.String("Average"), aws.String("Minimum"), aws.String("Maximum")},
Namespace: aws.String("AWS/ELB"),
Unit: aws.String("Seconds"),
Dimensions: []*cloudwatch.Dimension{{Name: aws.String("LoadBalancerName"), Value: loadBalancer.LoadBalancerName}},
}
resp, err := cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsELBLatencyMin, datapoint.Timestamp.Unix(), *datapoint.Minimum, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Second, descAWSELBLatency)
AddTS(md, awsELBLatencyMax, datapoint.Timestamp.Unix(), *datapoint.Maximum, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Second, descAWSELBLatency)
AddTS(md, awsELBLatencyAvg, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"loadbalancer": *loadBalancer.LoadBalancerName}, metadata.Gauge, metadata.Second, descAWSELBLatency)
}
return nil
}
func putMetric(metricName string, watchService *cloudwatch.CloudWatch, unitName string, metricValue float64) {
if watchService == nil {
return
}
params := &cloudwatch.PutMetricDataInput{
MetricData: []*cloudwatch.MetricDatum{
&cloudwatch.MetricDatum{
MetricName: aws.String(metricName),
Timestamp: aws.Time(time.Now()),
Unit: aws.String(unitName),
Value: aws.Float64(metricValue),
},
},
Namespace: aws.String(cloudwatchNamespace),
}
_, err := watchService.PutMetricData(params)
if err != nil {
log.Printf("Failure to put cloudwatch metric: %s", err)
}
}
func awsGetCPU(cw cloudwatch.CloudWatch, md *opentsdb.MultiDataPoint, instance *ec2.Instance) error {
search := cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(time.Now().UTC().Add(time.Second * -600)),
EndTime: aws.Time(time.Now().UTC()),
MetricName: aws.String("CPUUtilization"),
Period: &aws_period,
Statistics: []*string{aws.String("Average")},
Namespace: aws.String("AWS/EC2"),
Unit: aws.String("Percent"),
Dimensions: []*cloudwatch.Dimension{{Name: aws.String("InstanceId"), Value: instance.InstanceId}},
}
resp, err := cw.GetMetricStatistics(&search)
if err != nil {
return err
}
tags := opentsdb.TagSet{
"instance": *instance.InstanceId,
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsCPU, datapoint.Timestamp.Unix(), *datapoint.Average, tags, metadata.Gauge, metadata.Pct, descAWSEC2CPU)
}
return nil
}
func awsGetStatusChecks(cw cloudwatch.CloudWatch, md *opentsdb.MultiDataPoint, instance *ec2.Instance) error {
period := int64(60)
search := cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(time.Now().UTC().Add(time.Second * -60)),
EndTime: aws.Time(time.Now().UTC()),
MetricName: aws.String("StatusCheckFailed"),
Period: &period,
Statistics: []*string{aws.String("Average")},
Namespace: aws.String("AWS/EC2"),
Unit: aws.String("Count"),
Dimensions: []*cloudwatch.Dimension{{Name: aws.String("InstanceId"), Value: instance.InstanceId}},
}
resp, err := cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsStatusCheckFailed, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"instance": *instance.InstanceId}, metadata.Gauge, metadata.Count, descAWSEC2StatusCheck)
}
search.MetricName = aws.String("StatusCheckFailed_Instance")
resp, err = cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsStatusCheckFailed, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"instance": *instance.InstanceId, "category": "instance"}, metadata.Gauge, metadata.Count, descAWSEC2StatusCheck)
}
search.MetricName = aws.String("StatusCheckFailed_System")
resp, err = cw.GetMetricStatistics(&search)
if err != nil {
return err
}
for _, datapoint := range resp.Datapoints {
AddTS(md, awsStatusCheckFailed, datapoint.Timestamp.Unix(), *datapoint.Average, opentsdb.TagSet{"instance": *instance.InstanceId, "category": "system"}, metadata.Gauge, metadata.Count, descAWSEC2StatusCheck)
}
return nil
}