Exemple #1
0
// MakeSaveCommandHook takes a fuction based on the command name and will run it on save passing files, user,
// message, args... as arguments to the command. For the SaveHook function that is returned, If the command fails
// to execute or returns a non normal output then an error is returned.
func MakeSaveCommandHook(cmdName string) (f SaveHook, err error) {
	_, err = exec.LookPath(cmdName)
	if err != nil {
		return f, fmt.Errorf("command %v not found, failed to create save hook: %v", cmdName, err)
	}
	f = func(files, user, message string, args ...string) error {
		cArgs := []string{files, user, message}
		cArgs = append(cArgs, args...)
		slog.Infof("executing save hook %v\n", cmdName)
		c := exec.Command(cmdName, cArgs...)
		var cOut bytes.Buffer
		var cErr bytes.Buffer
		c.Stdout = &cOut
		c.Stderr = &cErr
		err := c.Start()
		if err != nil {
			return err
		}
		err = c.Wait()
		if err != nil {
			slog.Warning(cErr.String())
			return err
		}
		slog.Infof("save hook ouput: %v\n", cOut.String())
		return nil
	}
	return
}
Exemple #2
0
func c_aws(accessKey, secretKey, region string, billingEnabled bool) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
	conf := &aws.Config{
		Credentials: creds,
		Region:      &region,
	}
	ecc := ec2.New(session.New(), conf)
	if ecc == nil {
		return nil, fmt.Errorf("unable to login to EC2")
	}
	elb := elb.New(session.New(), conf)
	if elb == nil {
		return nil, fmt.Errorf("unable to login to ELB")
	}
	cw := cloudwatch.New(session.New(), conf)
	if cw == nil {
		return nil, fmt.Errorf("unable to login to CloudWatch")
	}
	instances, err := awsGetInstances(*ecc)
	if err != nil && !billingEnabled {
		slog.Warning("No EC2 Instances found.")
	}
	loadBalancers, err := awsGetLoadBalancers(*elb)
	if err != nil && !billingEnabled {
		slog.Warning("No ELB Load Balancers found.")
	}
	for _, loadBalancer := range loadBalancers {
		awsGetELBLatency(*cw, &md, loadBalancer)
		awsGetELBHostCounts(*cw, &md, loadBalancer)
	}
	for _, instance := range instances {
		awsGetCPU(*cw, &md, instance)
		awsGetNetwork(*cw, &md, instance)
		awsGetDiskBytes(*cw, &md, instance)
		awsGetDiskOps(*cw, &md, instance)
		awsGetStatusChecks(*cw, &md, instance)
	}
	return md, nil
}
Exemple #3
0
func (awsBilling *awsBillingConfig) Check() (opentsdb.MultiDataPoint, error) {
	md := opentsdb.MultiDataPoint{}
	purgeObjects := []*s3.Object{}
	//Declare the objects we want to fetch here. For completeness sake, we're going to fetch
	//the entire contents of the bucket, as we are going to be cleaning it out as we go.
	getBucketObjects := &s3.ListObjectsInput{
		Bucket: aws.String(awsBilling.bucketName),
	}
	//Get the objects from the bucket
	bucketObjects, err := awsBilling.s3svc.ListObjects(getBucketObjects)
	if err != nil {
		return nil, err
	}

	//Sort the files found by last modified, newest first.
	sort.Sort(sort.Reverse(ByLastModified(bucketObjects.Contents)))

	//Go through the contents of the bucket and parse the different kinds of report files
	//that that we're going to have. At the end of it we'll have a bucketContents that contains
	//the origina S3 file, along with hopefully enough other detail.
	var thisBucketContents []bucketContents
	for _, bucketObject := range bucketObjects.Contents {
		thisReport := &billingKeyStructure{}
		thisReport.parseFromObjectKey(bucketObject.Key, awsBilling)
		thisBucketContents = append(thisBucketContents, bucketContents{
			origS3Item:     bucketObject,
			awsBillingItem: thisReport,
		})
		//Mark files past the purge date to be purged
		if bucketObject.LastModified.Before(awsBilling.purgeOlderThan) {
			purgeObjects = append(purgeObjects, bucketObject)
		}
	}

	//So, billing comes with a bunch of different manifets and whatnot. We don't really need all of that
	//for this basic billing integration, as the first line of the CSVs we're interested in also has
	//the breakdown of how it's all structured. We only want to process the last file that is present
	//for each month, as each month we can get 30+ files. The most recent file for any given month should
	//contain everything we need.
	var monthsProcessed []string //The months that have been processed so far
	var allBills []*billHeader
	for _, billingObject := range thisBucketContents {
		isGZ := filepath.Ext(billingObject.awsBillingItem.fileName) == ".gz"                                              //If the filename ends in a .gz then it's probably what we want
		isBillingPath := strings.Contains(billingObject.awsBillingItem.fileName, billingObject.awsBillingItem.reportName) //If it ends in .gz and has the report name in it, then it's definately what we want
		notProcessedMonth := !stringInSlice(billingObject.origS3Item.LastModified.Format("200601"), monthsProcessed)      //Check if we have processed this month's file already
		if isGZ && isBillingPath && notProcessedMonth {
			//Download the file and un-gzip it in one step
			billingFile, err := billingObject.downloadGzippedItem(awsBilling.downloader, awsBilling)
			if err != nil {
				return nil, err
			}
			allBills = append(allBills, awsBilling.ReadBillingFile(billingFile))
			monthsProcessed = append(monthsProcessed, billingObject.origS3Item.LastModified.Format("200601")) //Log that we've processed this month
		}
	}
	//Dump into scollector format and clean up
	for _, thisBill := range allBills {
		thisBill.toTSDB(&md)
	}
	//Clean up after ourselves
	for _, purge := range purgeObjects {
		_, err := awsBilling.s3svc.DeleteObject(&s3.DeleteObjectInput{
			Bucket: aws.String(awsBilling.bucketName),
			Key:    purge.Key,
		})
		if err != nil {
			slog.Warning("Error deleting object:", err)
		}
	}
	return md, nil
}