func getVolumes() chan *reaperaws.Volume { ch := make(chan *reaperaws.Volume) go func() { volumeCh := reaperaws.AllVolumes() regionSums := make(map[reapable.Region]int) volumeSizeSums := make(map[reapable.Region]map[int64]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for volume := range volumeCh { // make the map if it is not initialized if volumeSizeSums[volume.Region()] == nil { volumeSizeSums[volume.Region()] = make(map[int64]int) } regionSums[volume.Region()]++ if isWhitelisted(volume) { whitelistedCount[volume.Region()]++ } volumeSizeSums[volume.Region()][*volume.Size]++ if matchesFilters(volume) { filteredCount[volume.Region()]++ } ch <- volume } for region, sum := range regionSums { log.Info("Found %d total volumes in %s", sum, region) } go func() { for region, regionMap := range volumeSizeSums { for volumeType, volumeSizeSum := range regionMap { err := reaperevents.NewStatistic("reaper.volumes.total", float64(volumeSizeSum), []string{fmt.Sprintf("region:%s,volumesize:%d", region, volumeType)}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.volumes.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s,volumesize:%d", region, volumeType)}) if err != nil { log.Error(err.Error()) } } err := reaperevents.NewStatistic("reaper.volumes.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region)}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
func getSecurityGroups() chan *reaperaws.SecurityGroup { ch := make(chan *reaperaws.SecurityGroup) go func() { securityGroupCh := reaperaws.AllSecurityGroups() regionSums := make(map[reapable.Region]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for sg := range securityGroupCh { regionSums[sg.Region()]++ if isWhitelisted(sg) { whitelistedCount[sg.Region()]++ } if matchesFilters(sg) { filteredCount[sg.Region()]++ } ch <- sg } for region, sum := range regionSums { log.Info("Found %d total SecurityGroups in %s", sum, region) } go func() { for region, regionSum := range regionSums { err := reaperevents.NewStatistic("reaper.securitygroups.total", float64(regionSum), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.securitygroups.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.securitygroups.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
// AllSecurityGroups describes every instance in the requested regions // *SecurityGroups are created for each *ec2.SecurityGroup // and are passed to a channel func AllSecurityGroups() chan *SecurityGroup { ch := make(chan *SecurityGroup, len(config.Regions)) // waitgroup for all regions wg := sync.WaitGroup{} for _, region := range config.Regions { wg.Add(1) go func(region string) { defer wg.Done() // add region to waitgroup api := ec2.New(sess, aws.NewConfig().WithRegion(region)) resp, err := api.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{}) for _, sg := range resp.SecurityGroups { ch <- NewSecurityGroup(region, sg) } if err != nil { // probably should do something here... log.Error(err.Error()) } }(region) } go func() { // in a separate goroutine, wait for all regions to finish // when they finish, close the chan wg.Wait() close(ch) }() return ch }
func getCloudformations() chan *reaperaws.Cloudformation { ch := make(chan *reaperaws.Cloudformation) go func() { cfs := reaperaws.AllCloudformations() regionSums := make(map[reapable.Region]int) filteredCount := make(map[reapable.Region]int) whitelistedCount := make(map[reapable.Region]int) for cf := range cfs { if isWhitelisted(cf) { whitelistedCount[cf.Region()]++ } regionSums[cf.Region()]++ if matchesFilters(cf) { filteredCount[cf.Region()]++ } ch <- cf } for region, sum := range regionSums { log.Info("Found %d total Cloudformation Stacks in %s", sum, region) } go func() { for region, regionSum := range regionSums { err := reaperevents.NewStatistic("reaper.cloudformations.total", float64(regionSum), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.cloudformations.filtered", float64(filteredCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } err = reaperevents.NewStatistic("reaper.cloudformations.whitelistedCount", float64(whitelistedCount[region]), []string{fmt.Sprintf("region:%s", region), config.EventTag}) if err != nil { log.Error(err.Error()) } } }() close(ch) }() return ch }
// matchesFilters applies the relevant filter groups to a filterable func matchesFilters(filterable filters.Filterable) bool { // recover from potential panics caused by malformed filters defer func() { if r := recover(); r != nil { log.Error("Recovered in matchesFilters with panic: ", r) } }() var groups map[string]filters.FilterGroup switch filterable.(type) { case *reaperaws.Instance: groups = config.Instances.FilterGroups case *reaperaws.AutoScalingGroup: groups = config.AutoScalingGroups.FilterGroups case *reaperaws.Cloudformation: groups = config.Cloudformations.FilterGroups case *reaperaws.SecurityGroup: groups = config.SecurityGroups.FilterGroups case *reaperaws.Volume: groups = config.Volumes.FilterGroups default: log.Warning("You probably screwed up and need to make sure matchesFilters works!") return false } matched := false // if there are no filters groups defined default to not match if len(groups) == 0 { return false } shouldFilter := false for _, group := range groups { if len(group) > 0 { // there is a filter shouldFilter = true } } // no filters, default to not match if !shouldFilter { return false } for name, group := range groups { didMatch := filters.ApplyFilters(filterable, group) if didMatch { matched = true filterable.AddFilterGroup(name, group) } } // convenient if isWhitelisted(filterable) { matched = false } return matched }
// AWSConsoleURL returns the url that can be used to access the resource on the AWS Console func (a *AutoScalingGroup) AWSConsoleURL() *url.URL { url, err := url.Parse(fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:id=%s;view=details", a.Region().String(), a.Region().String(), url.QueryEscape(a.ID().String()))) if err != nil { log.Error("Error generating AWSConsoleURL. ", err) } return url }
// AWSConsoleURL returns the url that can be used to access the resource on the AWS Console func (a *Instance) AWSConsoleURL() *url.URL { url, err := url.Parse(fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/v2/home?region=%s#Instances:instanceId=%s", a.Region().String(), a.Region().String(), url.QueryEscape(a.ID().String()))) if err != nil { log.Error("Error generating AWSConsoleURL. ", err) } return url }
func (filter *Filter) BoolValue(v int) (bool, error) { b, err := strconv.ParseBool(filter.Arguments[v]) if err != nil { log.Error(fmt.Sprintf("could not parse %s as bool", filter.Arguments[v])) return false, err } return b, nil }
func (filter *Filter) Int64Value(v int) (int64, error) { // parseint -> base 10, 64 bit int i, err := strconv.ParseInt(filter.Arguments[v], 10, 64) if err != nil { log.Error(fmt.Sprintf("could not parse %s as int64", filter.Arguments[v])) return 0, err } return i, nil }
// AWSConsoleURL returns the url that can be used to access the resource on the AWS Console func (a *Cloudformation) AWSConsoleURL() *url.URL { url, err := url.Parse("https://console.aws.amazon.com/cloudformation/home") // setting RawQuery because QueryEscape messes with the "/"s in the url url.RawQuery = fmt.Sprintf("region=%s#/stacks?filter=active&tab=overview&stackId=%s", a.Region().String(), a.ID().String()) if err != nil { log.Error("Error generating AWSConsoleURL. ", err) } return url }
func (e *Datadog) getGodspeed() { var gs *godspeed.Godspeed var err error // if config options not set, use defaults if e.Config.Host == "" || e.Config.Port == "" { gs, err = godspeed.NewDefault() } else { port, err := strconv.Atoi(e.Config.Port) if err != nil { log.Error(err.Error()) } gs, err = godspeed.New(e.Config.Host, port, false) } if err != nil { log.Error(err.Error()) } e._godspeed = gs }
// Ready NEEDS to be called for EventReporters and Reapables to be properly initialized // which means events AND config need to be set BEFORE Ready func Ready() { reaperevents.SetDryRun(config.DryRun) if r := reapable.NewReapables(config.AWS.Regions); r != nil { reapables = *r } else { log.Error("reapables improperly initialized") } }
// MakeWhitelistLink creates a tokenized link for whitelisting func makeWhitelistLink(region reapable.Region, id reapable.ID, tokenSecret, apiURL string) (string, error) { whitelist, err := token.Tokenize(tokenSecret, token.NewWhitelistJob(region.String(), id.String())) if err != nil { log.Error("Error creating whitelist link: ", err) return "", err } return makeURL(apiURL, "whitelist", whitelist), nil }
func Cleanup() { for _, er := range *eventReporters { c, ok := er.(Cleaner) if ok { if err := c.Cleanup(); err != nil { log.Error(err.Error()) } } } }
// MakeStopLink creates a tokenized link for stopping func makeStopLink(region reapable.Region, id reapable.ID, tokenSecret, apiURL string) (string, error) { stop, err := token.Tokenize(tokenSecret, token.NewStopJob(region.String(), id.String())) if err != nil { log.Error("Error creating ScaleToZero link: ", err) return "", err } return makeURL(apiURL, "stop", stop), nil }
func GetPrices() { // prevent shadowing var err error log.Info("Downloading prices") pricesMap, err = prices.DownloadPricesMap(prices.Ec2PricingUrl) if err != nil { log.Error(fmt.Sprintf("Error getting prices: %s", err.Error())) return } log.Info("Successfully downloaded prices") }
func (r *Reaper) reap() { reapables := allReapables() filteredOwnerMap := make(map[string][]reaperevents.Reapable) for _, reapable := range reapables { // default owner should ensure this does not happen if reapable.Owner() == nil { log.Error("Resource %s has no owner", reapable.ReapableDescriptionTiny()) continue } // TODO naively re-call matchesFilters here // after previously calling it for statistics if matchesFilters(reapable) { // group resources by owner owner := reapable.Owner().Address filteredOwnerMap[owner] = append(filteredOwnerMap[owner], reapable) registerReapable(reapable) } } // trigger batch events for each filtered owned resource in a goroutine // for each owner in the owner map go func() { // trigger a per owner batch event for _, filteredOwnedReapables := range filteredOwnerMap { // if there's only one resource for the owner, do a single event if len(filteredOwnedReapables) == 1 { if err := reaperevents.NewReapableEvent(filteredOwnedReapables[0], []string{config.EventTag}); err != nil { log.Error(err.Error()) } } else { // batch event if err := reaperevents.NewBatchReapableEvent(filteredOwnedReapables, []string{config.EventTag}); err != nil { log.Error(err.Error()) } } } }() }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *AutoScalingGroup) Terminate() (bool, error) { log.Info("Terminating AutoScalingGroup %s", a.ReapableDescriptionTiny()) as := autoscaling.New(session.New(&aws.Config{Region: aws.String(a.Region().String())})) input := &autoscaling.DeleteAutoScalingGroupInput{ AutoScalingGroupName: aws.String(a.ID().String()), } _, err := as.DeleteAutoScalingGroup(input) if err != nil { log.Error("could not delete AutoScalingGroup ", a.ReapableDescriptionTiny()) return false, err } return true, nil }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *Cloudformation) Terminate() (bool, error) { log.Info("Terminating Cloudformation %s", a.ReapableDescriptionTiny()) as := cloudformation.New(sess, aws.NewConfig().WithRegion(a.Region().String())) input := &cloudformation.DeleteStackInput{ StackName: aws.String(a.ID().String()), } _, err := as.DeleteStack(input) if err != nil { log.Error("could not delete Cloudformation ", a.ReapableDescriptionTiny()) return false, err } return false, nil }
// Terminate is a method of reapable.Terminable, which is embedded in reapable.Reapable func (a *SecurityGroup) Terminate() (bool, error) { log.Info("Terminating SecurityGroup ", a.ReapableDescriptionTiny()) api := ec2.New(sess, aws.NewConfig().WithRegion(string(a.Region()))) input := &ec2.DeleteSecurityGroupInput{ GroupName: aws.String(a.ID().String()), } _, err := api.DeleteSecurityGroup(input) if err != nil { log.Error("could not delete SecurityGroup ", a.ReapableDescriptionTiny()) return false, err } return false, nil }
func populatePricesMap(r io.Reader) (PricesMap, error) { defer func() { if r := recover(); r != nil { log.Error("Recovered from a panic: ", r) } }() // initialize inner map pricesMap := make(PricesMap) for _, region := range regions { pricesMap[region] = make(map[string]string) } pd := new(PriceData) err := json.NewDecoder(r).Decode(pd) if err != nil { return PricesMap{}, err } for sku, productData := range pd.Products { // only get prices for EC2 instances if productData.ProductFamily != "Compute Instance" { continue } for _, termData := range pd.Terms.OnDemand[sku] { for _, dimensionData := range termData.PriceDimensions { if region, ok := regions[productData.Attributes.Location]; ok { pricesMap[region][productData.Attributes.InstanceType] = dimensionData.PricePerUnit.USD } else { log.Error(fmt.Sprintf("Region not found for sku %s location %s", sku, productData.Attributes.Location)) } } } } return pricesMap, nil }
// Stop by region, id, calls a Reapable's own Stop method func Stop(region reapable.Region, id reapable.ID) error { reapable, err := reapables.Get(region, id) if err != nil { return err } _, err = reapable.Stop() if err != nil { log.Error(fmt.Sprintf("Could not stop resource with region: %s and id: %s. Error: %s", region, id, err.Error())) return err } log.Debug("Stop ", reapable.ReapableDescriptionShort()) return nil }
func (a *AutoScalingGroup) scaleToSize(size int64, minSize int64) (bool, error) { log.Info("Scaling AutoScalingGroup %s to size %d.", a.ReapableDescriptionTiny(), size) as := autoscaling.New(session.New(&aws.Config{Region: aws.String(a.Region().String())})) input := &autoscaling.UpdateAutoScalingGroupInput{ AutoScalingGroupName: aws.String(a.ID().String()), DesiredCapacity: &size, MinSize: &minSize, } _, err := as.UpdateAutoScalingGroup(input) if err != nil { log.Error("could not update AutoScalingGroup ", a.ReapableDescriptionTiny()) return false, err } return true, nil }
// cloudformationResources returns a chan of CloudformationResources, sourced from the AWS API // there is rate limiting in the AWS API for CloudformationResources, so we delay // this is skippable with the CLI flag -withoutCloudformationResources func cloudformationResources(region, id string) chan *cloudformation.StackResource { ch := make(chan *cloudformation.StackResource) if config.WithoutCloudformationResources { close(ch) return ch } api := cloudformation.New(sess, aws.NewConfig().WithRegion(region)) go func() { <-timeout // this query can fail, so we retry didRetry := false input := &cloudformation.DescribeStackResourcesInput{StackName: &id} // initial query resp, err := api.DescribeStackResources(input) for err != nil { sleepTime := 2*time.Second + time.Duration(rand.Intn(2000))*time.Millisecond if err != nil { // this error is annoying and will come up all the time... so you can disable it if strings.Split(err.Error(), ":")[0] == "Throttling" && log.Extras() { log.Warning("StackResources: %s (retrying %s after %ds)", err.Error(), id, sleepTime*1.0/time.Second) } else if strings.Split(err.Error(), ":")[0] != "Throttling" { // any other errors log.Error(fmt.Sprintf("StackResources: %s (retrying %s after %ds)", err.Error(), id, sleepTime*1.0/time.Second)) } } // wait a random amount of time... hopefully long enough to beat rate limiting time.Sleep(sleepTime) // retry query resp, err = api.DescribeStackResources(input) didRetry = true } if didRetry && log.Extras() { log.Info("Retry succeeded for %s!", id) } for _, resource := range resp.StackResources { ch <- resource } close(ch) }() return ch }
func makeURL(host, action, token string) string { if host == "" { log.Error("makeURL: host is empty") } action = url.QueryEscape(action) token = url.QueryEscape(token) vals := url.Values{} vals.Add(config.HTTP.Action, action) vals.Add(config.HTTP.Token, token) if host[len(host)-1:] == "/" { return host + "?" + vals.Encode() } return host + "/?" + vals.Encode() }
func main() { // config and events are vars in the reaper package // they NEED to be set before a reaper.Reaper can be initialized reaper.SetConfig(&config) reaperevents.SetEvents(&eventReporters) if config.DryRun { log.Info("Dry run mode enabled, no events will be triggered. Enable Extras in Notifications for per-event DryRun notifications.") reaperevents.SetDryRun(config.DryRun) } // Ready() NEEDS to be called after BOTH SetConfig() and SetEvents() // it uses those values to set individual EventReporter config values // and to init the Reapables map reaper.Ready() // sets the config variable in Reaper's AWS package // this also NEEDS to be set before a Reaper can be started reaperaws.SetConfig(&config.AWS) // single instance of reaper reapRunner := reaper.NewReaper() // Run the reaper process reapRunner.Start() // run the HTTP server api := reaper.NewHTTPApi(config.HTTP) if err := api.Serve(); err != nil { log.Error(err.Error()) } else { // HTTP server successfully started c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill) // waiting for an Interrupt or Kill signal // this channel blocks until it receives one sig := <-c log.Info("Got signal %s, stopping services", sig.String()) log.Info("Stopping HTTP") api.Stop() log.Info("Stopping reaper runner") reapRunner.Stop() } }
// AllInstances describes every instance in the requested regions // *Instances are created for each *ec2.Instance // and are passed to a channel func AllInstances() chan *Instance { ch := make(chan *Instance, len(config.Regions)) // waitgroup for all regions wg := sync.WaitGroup{} for _, region := range config.Regions { wg.Add(1) go func(region string) { defer wg.Done() // add region to waitgroup api := ec2.New(sess, aws.NewConfig().WithRegion(region)) // DescribeInstancesPages does autopagination err := api.DescribeInstancesPages(&ec2.DescribeInstancesInput{}, func(resp *ec2.DescribeInstancesOutput, lastPage bool) bool { for _, res := range resp.Reservations { for _, instance := range res.Instances { ch <- NewInstance(region, instance) } } // if we are at the last page, we should not continue // the return value of this func is "shouldContinue" if lastPage { return false } return true }) if err != nil { // probably should do something here... log.Error(err.Error()) } }(region) } go func() { // in a separate goroutine, wait for all regions to finish // when they finish, close the chan wg.Wait() close(ch) }() return ch }
// newReapableEvent is a method of EventReporter func (e *ReaperEvent) newReapableEvent(r Reapable, tags []string) error { if e.Config.shouldTriggerFor(r) { var err error switch e.Config.Mode { case "Stop": _, err = r.Stop() log.Info("ReaperEvent: Stopping ", r.ReapableDescriptionShort()) NewEvent("Reaper: Stopping ", r.ReapableDescriptionShort(), nil, []string{}) NewCountStatistic("reaper.reapables.stopped", []string{r.ReapableDescriptionTiny()}) case "Terminate": _, err = r.Terminate() log.Info("ReaperEvent: Terminating ", r.ReapableDescriptionShort()) NewEvent("Reaper: Terminating ", r.ReapableDescriptionShort(), nil, []string{}) NewCountStatistic("reaper.reapables.terminated", []string{r.ReapableDescriptionTiny()}) default: log.Error(fmt.Sprintf("Invalid %s Mode %s", e.Config.Name, e.Config.Mode)) } if err != nil { return err } } return nil }
// AllCloudformations returns a chan of Cloudformations, sourced from the AWS API func AllCloudformations() chan *Cloudformation { ch := make(chan *Cloudformation, len(config.Regions)) // waitgroup for all regions wg := sync.WaitGroup{} for _, region := range config.Regions { wg.Add(1) go func(region string) { defer wg.Done() // add region to waitgroup api := cloudformation.New(sess, aws.NewConfig().WithRegion(region)) err := api.DescribeStacksPages(&cloudformation.DescribeStacksInput{}, func(resp *cloudformation.DescribeStacksOutput, lastPage bool) bool { for _, stack := range resp.Stacks { ch <- NewCloudformation(region, stack) } // if we are at the last page, we should not continue // the return value of this func is "shouldContinue" if lastPage { // on the last page, finish this region return false } return true }) if err != nil { // probably should do something here... log.Error(err.Error()) } }(region) } go func() { // in a separate goroutine, wait for all regions to finish // when they finish, close the chan wg.Wait() close(ch) }() return ch }
// Filter is part of the filter.Filterable interface func (a *AutoScalingGroup) Filter(filter filters.Filter) bool { matched := false // map function names to function calls switch filter.Function { case "SizeGreaterThan": if i, err := filter.Int64Value(0); err == nil && a.sizeGreaterThan(i) { matched = true } case "SizeLessThan": if i, err := filter.Int64Value(0); err == nil && a.sizeLessThan(i) { matched = true } case "SizeEqualTo": if i, err := filter.Int64Value(0); err == nil && a.sizeEqualTo(i) { matched = true } case "SizeLessThanOrEqualTo": if i, err := filter.Int64Value(0); err == nil && a.sizeLessThanOrEqualTo(i) { matched = true } case "SizeGreaterThanOrEqualTo": if i, err := filter.Int64Value(0); err == nil && a.sizeGreaterThanOrEqualTo(i) { matched = true } case "CreatedTimeInTheLast": d, err := time.ParseDuration(filter.Arguments[0]) if err == nil && a.CreatedTime != nil && time.Since(*a.CreatedTime) < d { matched = true } case "CreatedTimeNotInTheLast": d, err := time.ParseDuration(filter.Arguments[0]) if err == nil && a.CreatedTime != nil && time.Since(*a.CreatedTime) > d { matched = true } case "InCloudformation": if b, err := filter.BoolValue(0); err == nil && a.IsInCloudformation == b { matched = true } case "Region": for _, region := range filter.Arguments { if a.Region() == reapable.Region(region) { matched = true } } case "NotRegion": // was this resource's region one of those in the NOT list regionSpecified := false for _, region := range filter.Arguments { if a.Region() == reapable.Region(region) { regionSpecified = true } } if !regionSpecified { matched = true } case "Tagged": if a.Tagged(filter.Arguments[0]) { matched = true } case "NotTagged": if !a.Tagged(filter.Arguments[0]) { matched = true } case "TagNotEqual": if a.Tag(filter.Arguments[0]) != filter.Arguments[1] { matched = true } case "ReaperState": if a.reaperState.State.String() == filter.Arguments[0] { matched = true } case "NotReaperState": if a.reaperState.State.String() != filter.Arguments[0] { matched = true } case "Named": if a.Name == filter.Arguments[0] { matched = true } case "NotNamed": if a.Name != filter.Arguments[0] { matched = true } case "IsDependency": if b, err := filter.BoolValue(0); err == nil && a.Dependency == b { matched = true } case "NameContains": if strings.Contains(a.Name, filter.Arguments[0]) { matched = true } case "NotNameContains": if !strings.Contains(a.Name, filter.Arguments[0]) { matched = true } default: log.Error(fmt.Sprintf("No function %s could be found for filtering AutoScalingGroups.", filter.Function)) } return matched }