// MakeSaveCommandHook takes a fuction based on the command name and will run it on save passing files, user,
// message, args... as arguments to the command. For the SaveHook function that is returned, If the command fails
// to execute or returns a non normal output then an error is returned.
func MakeSaveCommandHook(cmdName string) (f SaveHook, err error) {
_, err = exec.LookPath(cmdName)
if err != nil {
return f, fmt.Errorf("command %v not found, failed to create save hook: %v", cmdName, err)
}
f = func(files, user, message string, args ...string) error {
cArgs := []string{files, user, message}
cArgs = append(cArgs, args...)
slog.Infof("executing save hook %v\n", cmdName)
c := exec.Command(cmdName, cArgs...)
var cOut bytes.Buffer
var cErr bytes.Buffer
c.Stdout = &cOut
c.Stderr = &cErr
err := c.Start()
if err != nil {
return err
}
err = c.Wait()
if err != nil {
slog.Warning(cErr.String())
return err
}
slog.Infof("save hook ouput: %v\n", cOut.String())
return nil
}
return
}
func c_aws(accessKey, secretKey, region string, billingEnabled bool) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
conf := &aws.Config{
Credentials: creds,
Region: ®ion,
}
ecc := ec2.New(session.New(), conf)
if ecc == nil {
return nil, fmt.Errorf("unable to login to EC2")
}
elb := elb.New(session.New(), conf)
if elb == nil {
return nil, fmt.Errorf("unable to login to ELB")
}
cw := cloudwatch.New(session.New(), conf)
if cw == nil {
return nil, fmt.Errorf("unable to login to CloudWatch")
}
instances, err := awsGetInstances(*ecc)
if err != nil && !billingEnabled {
slog.Warning("No EC2 Instances found.")
}
loadBalancers, err := awsGetLoadBalancers(*elb)
if err != nil && !billingEnabled {
slog.Warning("No ELB Load Balancers found.")
}
for _, loadBalancer := range loadBalancers {
awsGetELBLatency(*cw, &md, loadBalancer)
awsGetELBHostCounts(*cw, &md, loadBalancer)
}
for _, instance := range instances {
awsGetCPU(*cw, &md, instance)
awsGetNetwork(*cw, &md, instance)
awsGetDiskBytes(*cw, &md, instance)
awsGetDiskOps(*cw, &md, instance)
awsGetStatusChecks(*cw, &md, instance)
}
return md, nil
}
func (awsBilling *awsBillingConfig) Check() (opentsdb.MultiDataPoint, error) {
md := opentsdb.MultiDataPoint{}
purgeObjects := []*s3.Object{}
//Declare the objects we want to fetch here. For completeness sake, we're going to fetch
//the entire contents of the bucket, as we are going to be cleaning it out as we go.
getBucketObjects := &s3.ListObjectsInput{
Bucket: aws.String(awsBilling.bucketName),
}
//Get the objects from the bucket
bucketObjects, err := awsBilling.s3svc.ListObjects(getBucketObjects)
if err != nil {
return nil, err
}
//Sort the files found by last modified, newest first.
sort.Sort(sort.Reverse(ByLastModified(bucketObjects.Contents)))
//Go through the contents of the bucket and parse the different kinds of report files
//that that we're going to have. At the end of it we'll have a bucketContents that contains
//the origina S3 file, along with hopefully enough other detail.
var thisBucketContents []bucketContents
for _, bucketObject := range bucketObjects.Contents {
thisReport := &billingKeyStructure{}
thisReport.parseFromObjectKey(bucketObject.Key, awsBilling)
thisBucketContents = append(thisBucketContents, bucketContents{
origS3Item: bucketObject,
awsBillingItem: thisReport,
})
//Mark files past the purge date to be purged
if bucketObject.LastModified.Before(awsBilling.purgeOlderThan) {
purgeObjects = append(purgeObjects, bucketObject)
}
}
//So, billing comes with a bunch of different manifets and whatnot. We don't really need all of that
//for this basic billing integration, as the first line of the CSVs we're interested in also has
//the breakdown of how it's all structured. We only want to process the last file that is present
//for each month, as each month we can get 30+ files. The most recent file for any given month should
//contain everything we need.
var monthsProcessed []string //The months that have been processed so far
var allBills []*billHeader
for _, billingObject := range thisBucketContents {
isGZ := filepath.Ext(billingObject.awsBillingItem.fileName) == ".gz" //If the filename ends in a .gz then it's probably what we want
isBillingPath := strings.Contains(billingObject.awsBillingItem.fileName, billingObject.awsBillingItem.reportName) //If it ends in .gz and has the report name in it, then it's definately what we want
notProcessedMonth := !stringInSlice(billingObject.origS3Item.LastModified.Format("200601"), monthsProcessed) //Check if we have processed this month's file already
if isGZ && isBillingPath && notProcessedMonth {
//Download the file and un-gzip it in one step
billingFile, err := billingObject.downloadGzippedItem(awsBilling.downloader, awsBilling)
if err != nil {
return nil, err
}
allBills = append(allBills, awsBilling.ReadBillingFile(billingFile))
monthsProcessed = append(monthsProcessed, billingObject.origS3Item.LastModified.Format("200601")) //Log that we've processed this month
}
}
//Dump into scollector format and clean up
for _, thisBill := range allBills {
thisBill.toTSDB(&md)
}
//Clean up after ourselves
for _, purge := range purgeObjects {
_, err := awsBilling.s3svc.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(awsBilling.bucketName),
Key: purge.Key,
})
if err != nil {
slog.Warning("Error deleting object:", err)
}
}
return md, nil
}
